def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
#print "Visited", type(Visited)
# nodes to add are on the input
NewNodes = Snap.TIntV()
t1 = time.time()
for item in msglist:
name = sw.GetMsgName(item)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f input %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f reading %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
#print "len", Vec.Len()
# get new nodes, not visited before
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f compute %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
nnodes = int(sw.GetVar("nodes"))
#ds["count"] += NewNodes.Len()
# done, no new nodes
#if ds["count"] >= nnodes:
if NewNodes.Len() <= 0:
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#print "%s.%03d output start" % (time.ctime(t2)[11:19], tmsec)
#t1 = t2
Visited.GetVal(ds["start"]).Val = 0 # reset start node to 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
#for i in xrange(0, DistCount.Len()):
# print "dist", i, DistCount.GetVal(i).Val
#for snode in xrange(0,nnodes):
# distance = Visited.GetVal(snode).Val
# if not dcount.has_key(distance):
# dcount[distance] = 0
# dcount[distance] += 1
l = []
for i in xrange(0, DistCount.Len()):
if DistCount.GetVal(i).Val <= 0:
break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.Send(0, dmsgout, "2")
sw.flog.write("final %s %s\n" % (str(ds["start"]), str(distance)))
sw.flog.write("distances " + str(l) + "\n")
sw.flog.flush()
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f output done" % (
# time.ctime(t2)[11:19], tmsec, tdiff)
return
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
#for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.Send(i, Vec1, swsnap=True)
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1
Visited = ds["visit"]
#print "Visited", type(Visited)
# nodes to add are on the input
NewNodes = Snap.TIntV()
t1 = time.time()
for item in msglist:
name = sw.GetMsgName(item)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f input %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
distance = Vec.Last().Val + 1
Vec.DelLast()
# subtract the starting index
Snap.IncVal(Vec, -ds["first"])
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f reading %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
#print "len", Vec.Len()
# get new nodes, not visited before
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance);
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f compute %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
ds["dist"] = distance
nnodes = ds["range"]
ds["count"] += NewNodes.Len()
sw.flog.write("distance %d, new %d, count %d, nodes %d\n" % (
distance, NewNodes.Len(), ds["count"], nnodes))
sw.flog.flush()
# done, no new nodes
#if NewNodes.Len() <= 0:
sw.flog.write("testing %d %d %d\n" % (ds["count"], nnodes, ds["count"] >= nnodes))
sw.flog.flush()
if ds["count"] >= nnodes:
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#print "%s.%03d output start" % (time.ctime(t2)[11:19], tmsec)
#t1 = t2
sw.flog.write("sending finish output\n")
sw.flog.flush()
if ds["start"] >= 0:
# reset start node to 0
Visited.GetVal(ds["start"]-ds["first"]).Val = 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited,DistCount)
#for i in xrange(0, DistCount.Len()):
# print "dist", i, DistCount.GetVal(i).Val
#for snode in xrange(0,nnodes):
# distance = Visited.GetVal(snode).Val
# if not dcount.has_key(distance):
# dcount[distance] = 0
# dcount[distance] += 1
# get the maximum positive distance
maxdist = DistCount.Len()-1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.flog.write("maxdist %d\n" % (maxdist))
sw.flog.flush()
# collect the distances
l = []
for i in xrange(0, maxdist):
#if DistCount.GetVal(i).Val <= 0:
# break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.Send(0,dmsgout,"2")
sw.flog.write("final %s %s\n" % (str(ds["start"]), str(distance)))
sw.flog.write("distances " + str(l) + "\n")
sw.flog.flush()
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f output done" % (
# time.ctime(t2)[11:19], tmsec, tdiff)
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
Snap.IncVal(NewNodes, ds["first"])
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
#for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(distance)
sw.Send(i,Vec1,swsnap=True)
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1
Visited = ds["visit"]
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
distance = Vec.Last(
).Val + 1 # update current distance for fringe nodes
Vec.DelLast()
# subtract the starting index
Snap.IncVal(Vec, -ds["first"])
# print "len", Vec.Len()
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
ds["dist"] = distance
nnodes = ds["range"]
ds["count"] += NewNodes.Len()
sw.log.info("distance: %d, new: %d, count: %d, nodes: %d" % \
(distance, NewNodes.Len(), ds["count"], nnodes))
# done, no new nodes
sw.log.debug("testing: %d %d %d" % \
(ds["count"], nnodes, ds["count"] >= nnodes))
if ds["count"] >= nnodes:
sw.log.info("sending finish output")
if ds["start"] >= 0:
# reset start node to 0
Visited.GetVal(ds["start"] - ds["first"]).Val = 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
# get the maximum positive distance
maxdist = DistCount.Len() - 1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.log.info("maxdist: %d" % maxdist)
# collect the distances
l = []
for i in xrange(maxdist):
# if DistCount.GetVal(i).Val <= 0: break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0, dmsgout, "2")
sw.cum_timer.cum_stop("network")
sw.log.info("final: %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances: %s" % str(l))
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
Snap.IncVal(NewNodes, ds["first"])
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# send the messages
timer.start("dist-send-all")
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(distance)
sw.cum_timer.cum_start("network")
sw.Send(i, Vec1, swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# print "len", Vec.Len()
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
# done, no new nodes
if NewNodes.Len() <= 0:
Visited.GetVal(ds["start"]).Val = 0 # reset start node to 0
timer.start("dist-get-distribution")
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited,DistCount)
l = []
for i in xrange(0, DistCount.Len()):
if DistCount.GetVal(i).Val <= 0:
break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0,dmsgout,"2")
sw.cum_timer.cum_stop("network")
sw.log.info("final: %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances: %s" % str(l))
timer.stop("dist-get-distribution")
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
timer.start("dist-send-all")
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.cum_timer.cum_start("network")
sw.Send(i,Vec1,swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1 # this should get overwritten if we have messages.
# logger gives a warning if that doesn't happen
Visited = ds["visit"]
seg_bits = int(sw.GetVar('seg_bits'))
this_segment_start = Snap.zeroLowOrderBits(ds['first'], seg_bits)
sw.log.debug('this task starts at node %d' % ds['first'])
sw.log.debug('this segment starts at node %d' % this_segment_start)
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV(
) # TODO (smacke): I think this is fine non-segmented
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
FringeSubset = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# it's okay to reassign and then use this later outside of the loop
# since BSP should guarantee that this is the same at every loop iteration
distance = FringeSubset.Last(
).Val + 1 # last value has prev distance, so we inc by 1
FringeSubset.DelLast()
# subtract the starting index
sw.log.debug('[%s] offsetting by first node id' % sw.GetName())
Snap.IncVal(FringeSubset, -(ds["first"] - this_segment_start))
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
# NewNodes will each have the segmented bits zero'd out, as well as
# the high-order bits for this task
sw.log.debug('[%s] calling GetNewNodes1' % sw.GetName())
Snap.GetNewNodes1(FringeSubset, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
ds["dist"] = distance
# This should never be -1 after processing message
if distance == -1:
sw.log.warn("[%s] thinks that the current distance is -1, \
this almost certainly means that things are broken!" % sw.GetName())
nnodes = ds["range"]
ds["count"] += NewNodes.Len() # no. of new things we visited
sw.log.info("distance: %d, new: %d, count: %d, nodes: %d" % \
(distance, NewNodes.Len(), ds["count"], nnodes))
sw.log.debug("testing: %d %d %d" % \
(ds["count"], nnodes, ds["count"] >= nnodes))
# done, no new nodes
if ds["count"] >= nnodes:
sw.log.debug("sending finish output")
if ds["source"] >= 0:
# reset start node to 0
Visited.GetVal(ds["source"] -
ds["first"]).Val = 0 # SMACKE: should be ok
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
# get the maximum positive distance
maxdist = DistCount.Len() - 1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.log.debug("maxdist: %d" % maxdist)
# collect the distances
l = []
for i in xrange(maxdist):
# if DistCount.GetVal(i).Val <= 0: break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["source"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0, dmsgout, "2")
sw.cum_timer.cum_stop("network")
sw.log.debug("final: %s %s" % (str(ds["source"]), str(distance)))
sw.log.debug("distances: %s" % str(l))
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
#ntasks = int(sw.GetVar("gen_tasks"))
ntasks = (1 << seg_bits
) / tsize # reduce number to only tasks within same segment
Tasks = Snap.TIntIntVV(ntasks)
# we will only ever send to tasks in the same segment, but # TODO (smacke) not wasting space anymore
# the wasted space shouldn't hurt that much
sw.log.debug('[%s] increase nodes to within-segment values now' %
sw.GetName())
Snap.IncVal(NewNodes, ds["first"] - this_segment_start)
# assign nodes to tasks
sw.log.debug('[%s] calling Nodes2Tasks1' % sw.GetName())
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
# All of the GetNbr tasks are in the same segment as this task
# so this should still work; we just have to find the base task for
# this segment and add it to all of the task indexes in Tasks
timer.stop("dist-collect-nodes")
# send the messages
timer.start("dist-send-all")
for i in xrange(Tasks.Len()):
Vec1 = Tasks.GetVal(i)
sw.log.debug('Vec1 length: %d' % Vec1.Len())
if Vec1.Len() <= 0:
continue
# add task# at the end # TODO (smacke): I still don't understand this terminology
Vec1.Add(distance)
sw.cum_timer.cum_start("network")
# we need to send to the correct segment, from which our
# tasks are offset
sw.Send(i + this_segment_start / tsize, Vec1, swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")