def GenGraph(sw):
"""
generate the graph edges
"""
# extract the stubs from the args
# iterate through the input queue and add new items to the stub list
# taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.log.debug("msglist: %s" % msglist)
Stubs = Snap.TIntIntVV() # Stubs is an empty vector
for item in msglist:
# 1) Get item in msglist
# 2) Get name of item
name = sw.GetMsgName(item)
# 3) Get vector associated with name
FIn = Snap.TFIn(Snap.TStr(name))
Vec64 = Snap.TIntIntVV(FIn)
# 4) Add vector to Stubs
#Stubs.AddV(Vec)
Snap.AddVec64(Stubs, Vec64)
# 5) Got all stubs, which is of length msglist
# nodes in each task
tsize = sw.GetRange()
# number of bits in our segment (so seg size is (1<<seg_bits))
seg_bits = int(sw.GetVar('seg_bits'))
# number of tasks
ntasks = int(sw.GetVar("gen_tasks"))
# get edges for a specific task
Tasks = Snap.TIntVVV(ntasks) # vector of length ntasks containing vectors
sw.log.debug('[%s] about to assign random edges' % sw.GetName())
# handles shuffling and random assignment of edges
Snap.AssignRandomEdges64(Stubs, Tasks, tsize, seg_bits)
sw.log.debug('[%s] done assigning random edges' % sw.GetName())
# send messages
for i in xrange(0, Tasks.Len()):
sw.log.debug(LazyStr(lambda: '[%s] sending TIntIntVV of memory size %d to %d' % \
(sw.GetName(), Snap.GetMemSize64(Tasks.GetVal(i)), i)))
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def GetNbr(sw):
"""
provide graph neighbors
"""
taskname = sw.GetName()
# tindex = sw.GetIndex()
msglist = sw.GetMsgList()
sw.log.debug("msglist %s" % msglist)
with perf.Timer(sw.log, "LoadState-GetNbrCpp"):
AdjLists = LoadState(sw)
if AdjLists:
# state is available, process requests for neighbors
sw.log.debug('[%s] state available, length %d' %
(sw.GetName(), AdjLists.Len()))
for item in msglist:
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
msg = Snap.TIntV(FIn)
GetNeighbors(sw, AdjLists, msg)
return
# state not found, initialize it with neighbors
sw.log.debug('[%s] adjlist not found, initializing' % sw.GetName())
Edges = Snap.TIntIntVV()
for item in msglist:
name = sw.GetMsgName(item)
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntIntVV(FIn)
Snap.AddVec64(Edges, Vec)
# first iteration: input are edges, save the state
AdjLists = GetEdges(sw, Edges)
sw.log.debug('[%s] saving adjlist of size %d now' %
(sw.GetName(), AdjLists.Len()))
with perf.Timer(sw.log, "SaveState-GetNbrCpp"):
SaveState(sw, AdjLists)
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "targets"
dmsgout["body"] = {}
sw.Send(0, dmsgout, "2")
def GetNeighbors(sw, AdjLists, Nodes):
# report node neighbors
# the last value is the distance
dist = Nodes.Last().Val
Nodes.DelLast()
Hood = Snap.TIntV()
Snap.GetNeighborhood(Nodes, AdjLists, Hood)
print "Hood len %d" % (Hood.Len())
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("stat_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(Hood, Tasks, tsize)
# send the messages
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add distance at the end
Vec1.Add(dist)
sw.Send(i, Vec1, swsnap=True)
def GetNeighbors(sw, AdjLists, Nodes):
# report node neighbors
# the last value is the task
# TODO (smacke): ^ I don't understand this comment
dist = Nodes.Last().Val
Nodes.DelLast()
SegmentedHood = Snap.TIntIntVV()
sw.log.debug('getting neighbors for %d nodes', Nodes.Len())
Snap.GetNeighborhood64(Nodes, AdjLists, SegmentedHood)
sw.log.debug("SegmentedHood len: %d" % SegmentedHood.Len())
# TODO (smacke): This gets $drange, since
# we are sending this to the GetDist task. Is there
# any way to make this clearer in the config file?
# Perhaps add in optional string argument to GetRange()
# which takes a destination port number, so we don't
# specify GetNbr range as $drange in the config.
# Just to reiterate, this is confusing because the 'range'
# parameter for GetNbr does not correspond to the actual range
# of ndoes that a GetNbr task is responsible for -- it corresponds
# to the range of nodes for the task that GetNbr talks to.
tsize = sw.GetRange()
seg_bits = int(sw.GetVar('seg_bits'))
# collect nodes for the same task
ntasks = int(sw.GetVar("stat_tasks"))
Tasks = Snap.TIntIntVV(
ntasks
) # this should be fine as long as (1<<seg_bits) is a multiple of drange
# assign nodes to tasks
Snap.Nodes2Tasks64(SegmentedHood, Tasks, tsize, seg_bits)
# send the messages
for i in xrange(Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add distance at the end
Vec1.Add(dist)
sw.Send(i, Vec1, swsnap=True)
def GenStubs(sw):
"""
determine degrees for all the nodes, generate the stubs and distribute them
"""
taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.flog.write("msglist " + str(msglist) + "\n")
sw.flog.flush()
ntasks = int(sw.GetVar("gen_tasks"))
sw.flog.write("__tasks__ %s\t%s\n" % (taskname, str(ntasks)))
sw.flog.flush()
for item in msglist:
dmsg = sw.GetMsg(item)
d = dmsg["body"]
sw.flog.write("task %s, args %s\n" % (taskname, str(d)))
sw.flog.flush()
ns = d["s"]
nrange = d["r"]
sw.flog.write("task %s, start %d, range %d\n" % (taskname, ns, nrange))
sw.flog.flush()
# determine node degrees
DegV = Snap.TIntV(nrange)
Snap.GetDegrees(DegV, distmean, distvar)
sw.flog.write("1 got degrees\n")
sw.flog.flush()
# randomly assign stubs to tasks
Tasks = Snap.TIntIntVV(ntasks)
Snap.AssignRndTask(DegV, Tasks)
sw.flog.write("2 assigned stubs\n")
sw.flog.flush()
# add ns to all values in Tasks
for i in range(0, Tasks.Len()):
Snap.IncVal(Tasks.GetVal(i), ns)
sw.flog.write("3 incremented base\n")
sw.flog.flush()
# send messages
for i in range(0, Tasks.Len()):
#sw.flog.write("sending task %d" % (i) + "\n")
sw.flog.write("sending task %d, len %d" %
(i, Tasks.GetVal(i).Len()) + "\n")
sw.flog.flush()
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def cmp(gname, size):
G = snap.LoadEdgeList(snap.PUNGraph, gname, 0, 1)
print "G: Nodes %d, Edges %d" % (G.GetNodes(), G.GetEdges())
Communities = snap.TIntIntVV()
snap.TCliqueOverlap_GetCPMCommunities(G, size, Communities)
print "---------------"
for C in Communities:
for I in C:
print I
print "---------------"
def GenGraph(sw):
"""
generate the graph edges
"""
# extract the stubs from the args
# iterate through the input queue and add new items to the stub list
# taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.log.debug("msglist: %s" % msglist)
Stubs = Snap.TIntV() # Stubs is an empty vector
for item in msglist:
# 1) Get item in msglist
# 2) Get name of item
name = sw.GetMsgName(item)
# 3) Get vector associated with name
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
# 4) Add vector to Stubs
Stubs.AddV(Vec)
# 5) Got all stubs, which is of length msglist
# Randomize the items (aka shuffle)
Snap.Randomize(Stubs)
# nodes in each task and the number of tasks
tsize = sw.GetRange()
ntasks = int(sw.GetVar("gen_tasks"))
# get edges for a specific task
Tasks = Snap.TIntIntVV(
ntasks) # vector of length ntasks containing vectors
Snap.AssignEdges(Stubs, Tasks, tsize)
# send messages
for i in xrange(0, Tasks.Len()):
sw.log.debug("sending task: %d, len: %d" % (i, Tasks.GetVal(i).Len()))
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def GenStubs(sw):
"""
determine degrees for all the nodes, generate the stubs and distribute them
"""
taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.log.debug("msglist: %s" % str(msglist))
ntasks = int(sw.GetVar("gen_tasks"))
sw.log.debug("__tasks__ %s\t%d" % (taskname, ntasks))
for item in msglist:
dmsg = sw.GetMsg(item)
d = dmsg["body"]
ns = d["s"]
nrange = d["r"]
sw.log.debug("task: %s, args: %s, start: %d, range: %d" % \
(taskname, str(d), ns, nrange))
# 1) Get degrees
# determine node degrees
DegV = Snap.TIntV(nrange) # vector of vertices' degree
Snap.GetDegrees(DegV, distmean, distvar) # populate
# 2) Assign stubs
# randomly assign stubs to tasks
Tasks = Snap.TIntIntVV(
ntasks) # vector of tasks, with each cell a vector
Snap.AssignRndTask(DegV,
Tasks) # each task is assigned a vec of vertex ids
# 3) Incremented base (above)
# add ns to all values in Tasks
for i in xrange(0, Tasks.Len()):
# inc the values in each list by ns (num_start)
# so that they are true vectex ids
Snap.IncVal(Tasks.GetVal(i), ns)
# send messages
for i in xrange(0, Tasks.Len()):
sw.log.debug("sending task %d, len %d" %
(i, Tasks.GetVal(i).Len()))
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
#print "Visited", type(Visited)
# nodes to add are on the input
NewNodes = Snap.TIntV()
t1 = time.time()
for item in msglist:
name = sw.GetMsgName(item)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f input %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f reading %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
#print "len", Vec.Len()
# get new nodes, not visited before
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f compute %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
nnodes = int(sw.GetVar("nodes"))
#ds["count"] += NewNodes.Len()
# done, no new nodes
#if ds["count"] >= nnodes:
if NewNodes.Len() <= 0:
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#print "%s.%03d output start" % (time.ctime(t2)[11:19], tmsec)
#t1 = t2
Visited.GetVal(ds["start"]).Val = 0 # reset start node to 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
#for i in xrange(0, DistCount.Len()):
# print "dist", i, DistCount.GetVal(i).Val
#for snode in xrange(0,nnodes):
# distance = Visited.GetVal(snode).Val
# if not dcount.has_key(distance):
# dcount[distance] = 0
# dcount[distance] += 1
l = []
for i in xrange(0, DistCount.Len()):
if DistCount.GetVal(i).Val <= 0:
break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.Send(0, dmsgout, "2")
sw.flog.write("final %s %s\n" % (str(ds["start"]), str(distance)))
sw.flog.write("distances " + str(l) + "\n")
sw.flog.flush()
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f output done" % (
# time.ctime(t2)[11:19], tmsec, tdiff)
return
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
#for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.Send(i, Vec1, swsnap=True)
size += Vec2.Len()
for j in range(0, Vec2.Len()):
print i, Vec2.GetVal(j).Val
print "done", Edges.Len(), AdjLists.Len(), size, len(d)
sys.exit(0)
#print "dir(snap.TIntV)", dir(snap.TIntV)
Vec1 = snap.TIntV(numnodes)
#print "dir(Vec1)", dir(Vec1)
print "Len Vec1", Vec1.Len()
#print "dir(snap.TIntIntVV)", dir(snap.TIntIntVV)
Vec2 = snap.TIntIntVV(numtask)
#print "dir(Vec2)", dir(Vec2)
print "Len Vec2", Vec2.Len()
print "Vec1", type(Vec1)
snap.GetDegrees(Vec1, 10.0, 1.5)
for i in range(0, Vec1.Len()):
print "Vec1", i, Vec1.GetVal(i).Val
snap.AssignRndTask(Vec1, Vec2)
for i in range(0, Vec2.Len()):
Vec3 = Vec2.GetVal(i)
print "Vec3", i, Vec3.Len()
def GenGraph(sw):
"""
generate the graph edges
"""
# extract the stubs from the args
# iterate through the input queue and add new items to the stub list
taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.flog.write("msglist " + str(msglist) + "\n")
sw.flog.flush()
Stubs = Snap.TIntV()
for item in msglist:
sw.flog.write("1 got item " + item + "\n")
sw.flog.flush()
name = sw.GetMsgName(item)
sw.flog.write("2 got name " + name + "\n")
sw.flog.flush()
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.flog.write("3 got vector %d" % (Vec.Len()) + "\n")
sw.flog.flush()
Stubs.AddV(Vec)
#for i in range(0,Vec.Len()):
# stubs.append(Vec.GetVal(i).Val)
#sw.flog.write("4 got stubs %d" % (len(stubs)) + "\n")
sw.flog.write("4 got stubs %d" % (Stubs.Len()) + "\n")
sw.flog.flush()
sw.flog.write("5 got all stubs\n")
sw.flog.flush()
#print taskname,stubs
# randomize the items
Snap.Randomize(Stubs)
#random.shuffle(stubs)
#print taskname + "-r",stubs
# get the pairs
#pairs = zip(stubs[::2], stubs[1::2])
#print taskname,pairs
# nodes in each task and the number of tasks
tsize = sw.GetRange()
ntasks = int(sw.GetVar("gen_tasks"))
# get edges for a specific task
Tasks = Snap.TIntIntVV(ntasks)
Snap.AssignEdges(Stubs, Tasks, tsize)
#print taskname,edges
# send messages
for i in range(0, Tasks.Len()):
#sw.flog.write("sending task %d" % (i) + "\n")
sw.flog.write("sending task %d, len %d" % (i, Tasks.GetVal(i).Len()) +
"\n")
sw.flog.flush()
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1
Visited = ds["visit"]
#print "Visited", type(Visited)
# nodes to add are on the input
NewNodes = Snap.TIntV()
t1 = time.time()
for item in msglist:
name = sw.GetMsgName(item)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f input %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
distance = Vec.Last().Val + 1
Vec.DelLast()
# subtract the starting index
Snap.IncVal(Vec, -ds["first"])
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f reading %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
#print "len", Vec.Len()
# get new nodes, not visited before
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance);
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f compute %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
ds["dist"] = distance
nnodes = ds["range"]
ds["count"] += NewNodes.Len()
sw.flog.write("distance %d, new %d, count %d, nodes %d\n" % (
distance, NewNodes.Len(), ds["count"], nnodes))
sw.flog.flush()
# done, no new nodes
#if NewNodes.Len() <= 0:
sw.flog.write("testing %d %d %d\n" % (ds["count"], nnodes, ds["count"] >= nnodes))
sw.flog.flush()
if ds["count"] >= nnodes:
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#print "%s.%03d output start" % (time.ctime(t2)[11:19], tmsec)
#t1 = t2
sw.flog.write("sending finish output\n")
sw.flog.flush()
if ds["start"] >= 0:
# reset start node to 0
Visited.GetVal(ds["start"]-ds["first"]).Val = 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited,DistCount)
#for i in xrange(0, DistCount.Len()):
# print "dist", i, DistCount.GetVal(i).Val
#for snode in xrange(0,nnodes):
# distance = Visited.GetVal(snode).Val
# if not dcount.has_key(distance):
# dcount[distance] = 0
# dcount[distance] += 1
# get the maximum positive distance
maxdist = DistCount.Len()-1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.flog.write("maxdist %d\n" % (maxdist))
sw.flog.flush()
# collect the distances
l = []
for i in xrange(0, maxdist):
#if DistCount.GetVal(i).Val <= 0:
# break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.Send(0,dmsgout,"2")
sw.flog.write("final %s %s\n" % (str(ds["start"]), str(distance)))
sw.flog.write("distances " + str(l) + "\n")
sw.flog.flush()
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f output done" % (
# time.ctime(t2)[11:19], tmsec, tdiff)
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
Snap.IncVal(NewNodes, ds["first"])
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
#for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(distance)
sw.Send(i,Vec1,swsnap=True)
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1 # this should get overwritten if we have messages.
# logger gives a warning if that doesn't happen
Visited = ds["visit"]
seg_bits = int(sw.GetVar('seg_bits'))
this_segment_start = Snap.zeroLowOrderBits(ds['first'], seg_bits)
sw.log.debug('this task starts at node %d' % ds['first'])
sw.log.debug('this segment starts at node %d' % this_segment_start)
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV(
) # TODO (smacke): I think this is fine non-segmented
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
FringeSubset = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# it's okay to reassign and then use this later outside of the loop
# since BSP should guarantee that this is the same at every loop iteration
distance = FringeSubset.Last(
).Val + 1 # last value has prev distance, so we inc by 1
FringeSubset.DelLast()
# subtract the starting index
sw.log.debug('[%s] offsetting by first node id' % sw.GetName())
Snap.IncVal(FringeSubset, -(ds["first"] - this_segment_start))
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
# NewNodes will each have the segmented bits zero'd out, as well as
# the high-order bits for this task
sw.log.debug('[%s] calling GetNewNodes1' % sw.GetName())
Snap.GetNewNodes1(FringeSubset, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
ds["dist"] = distance
# This should never be -1 after processing message
if distance == -1:
sw.log.warn("[%s] thinks that the current distance is -1, \
this almost certainly means that things are broken!" % sw.GetName())
nnodes = ds["range"]
ds["count"] += NewNodes.Len() # no. of new things we visited
sw.log.info("distance: %d, new: %d, count: %d, nodes: %d" % \
(distance, NewNodes.Len(), ds["count"], nnodes))
sw.log.debug("testing: %d %d %d" % \
(ds["count"], nnodes, ds["count"] >= nnodes))
# done, no new nodes
if ds["count"] >= nnodes:
sw.log.debug("sending finish output")
if ds["source"] >= 0:
# reset start node to 0
Visited.GetVal(ds["source"] -
ds["first"]).Val = 0 # SMACKE: should be ok
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
# get the maximum positive distance
maxdist = DistCount.Len() - 1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.log.debug("maxdist: %d" % maxdist)
# collect the distances
l = []
for i in xrange(maxdist):
# if DistCount.GetVal(i).Val <= 0: break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["source"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0, dmsgout, "2")
sw.cum_timer.cum_stop("network")
sw.log.debug("final: %s %s" % (str(ds["source"]), str(distance)))
sw.log.debug("distances: %s" % str(l))
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
#ntasks = int(sw.GetVar("gen_tasks"))
ntasks = (1 << seg_bits
) / tsize # reduce number to only tasks within same segment
Tasks = Snap.TIntIntVV(ntasks)
# we will only ever send to tasks in the same segment, but # TODO (smacke) not wasting space anymore
# the wasted space shouldn't hurt that much
sw.log.debug('[%s] increase nodes to within-segment values now' %
sw.GetName())
Snap.IncVal(NewNodes, ds["first"] - this_segment_start)
# assign nodes to tasks
sw.log.debug('[%s] calling Nodes2Tasks1' % sw.GetName())
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
# All of the GetNbr tasks are in the same segment as this task
# so this should still work; we just have to find the base task for
# this segment and add it to all of the task indexes in Tasks
timer.stop("dist-collect-nodes")
# send the messages
timer.start("dist-send-all")
for i in xrange(Tasks.Len()):
Vec1 = Tasks.GetVal(i)
sw.log.debug('Vec1 length: %d' % Vec1.Len())
if Vec1.Len() <= 0:
continue
# add task# at the end # TODO (smacke): I still don't understand this terminology
Vec1.Add(distance)
sw.cum_timer.cum_start("network")
# we need to send to the correct segment, from which our
# tasks are offset
sw.Send(i + this_segment_start / tsize, Vec1, swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
size += Vec2.Len()
for j in range(0,Vec2.Len()):
print(i, Vec2.GetVal(j).Val)
print("done", Edges.Len(), AdjLists.Len(), size, len(d))
sys.exit(0)
#print("dir(Snap.TIntV)", dir(Snap.TIntV))
Vec1 = Snap.TIntV(numnodes)
#print("dir(Vec1)", dir(Vec1))
print("Len Vec1", Vec1.Len())
#print("dir(Snap.TIntIntVV)", dir(Snap.TIntIntVV))
Vec2 = Snap.TIntIntVV(numtask)
#print("dir(Vec2)", dir(Vec2))
print("Len Vec2", Vec2.Len())
print("Vec1", type(Vec1))
Snap.GetDegrees(Vec1, 10.0, 1.5)
for i in range(0,Vec1.Len()):
print("Vec1", i, Vec1.GetVal(i).Val)
Snap.AssignRndTask(Vec1, Vec2)
for i in range(0,Vec2.Len()):
Vec3 = Vec2.GetVal(i)
print("Vec3", i, Vec3.Len())
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
# print "Visited", type(Visited)
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntH()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# print "input", name
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# print "len", Vec.Len()
# get new nodes, not visited before
timer.start("dist-nodes-iter")
Snap.GetNewNodes(Vec, Visited, NewNodes, distance)
timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
# done, no new nodes
if NewNodes.Len() <= 0:
timer.start("dist-get-distribution")
# get distance distribution
dcount = {}
# TODO move this loop to SNAP C++
VIter = Visited.BegI()
while not VIter.IsEnd():
# snode = VIter.GetKey().Val
distance = VIter.GetDat().Val
if not dcount.has_key(distance):
dcount[distance] = 0
dcount[distance] += 1
VIter.Next()
nnodes = int(sw.GetVar("nodes"))
l = []
for i in xrange(0, nnodes):
if not dcount.has_key(i):
break
l.append(dcount[i])
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0,dmsgout,"2")
sw.cum_timer.cum_stop("network")
sw.log.info("final %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances %s" % str(l))
timer.stop("dist-get-distribution")
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
timer.start("dist-send-all")
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.cum_timer.cum_start("network")
sw.Send(i,Vec1,swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# print "len", Vec.Len()
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
# done, no new nodes
if NewNodes.Len() <= 0:
Visited.GetVal(ds["start"]).Val = 0 # reset start node to 0
timer.start("dist-get-distribution")
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited,DistCount)
l = []
for i in xrange(0, DistCount.Len()):
if DistCount.GetVal(i).Val <= 0:
break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0,dmsgout,"2")
sw.cum_timer.cum_stop("network")
sw.log.info("final: %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances: %s" % str(l))
timer.stop("dist-get-distribution")
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
timer.start("dist-send-all")
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.cum_timer.cum_start("network")
sw.Send(i,Vec1,swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1
Visited = ds["visit"]
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
distance = Vec.Last(
).Val + 1 # update current distance for fringe nodes
Vec.DelLast()
# subtract the starting index
Snap.IncVal(Vec, -ds["first"])
# print "len", Vec.Len()
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
ds["dist"] = distance
nnodes = ds["range"]
ds["count"] += NewNodes.Len()
sw.log.info("distance: %d, new: %d, count: %d, nodes: %d" % \
(distance, NewNodes.Len(), ds["count"], nnodes))
# done, no new nodes
sw.log.debug("testing: %d %d %d" % \
(ds["count"], nnodes, ds["count"] >= nnodes))
if ds["count"] >= nnodes:
sw.log.info("sending finish output")
if ds["start"] >= 0:
# reset start node to 0
Visited.GetVal(ds["start"] - ds["first"]).Val = 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
# get the maximum positive distance
maxdist = DistCount.Len() - 1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.log.info("maxdist: %d" % maxdist)
# collect the distances
l = []
for i in xrange(maxdist):
# if DistCount.GetVal(i).Val <= 0: break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0, dmsgout, "2")
sw.cum_timer.cum_stop("network")
sw.log.info("final: %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances: %s" % str(l))
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
Snap.IncVal(NewNodes, ds["first"])
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# send the messages
timer.start("dist-send-all")
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(distance)
sw.cum_timer.cum_start("network")
sw.Send(i, Vec1, swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")