def GVizTest(NNodes, NEdges):
Graph = snap.GenRndGnm(NNodes, NEdges, 1)
FName = "test.png"
snap.DrawGViz(Graph, 1, snap.TStr(FName),
snap.TStr("Snap Ringo Dot"), 1)
return os.path.exists(FName)
def GetNbr(sw):
"""
provide graph neighbors
"""
taskname = sw.GetName()
# tindex = sw.GetIndex()
msglist = sw.GetMsgList()
sw.log.debug("msglist %s" % msglist)
with perf.Timer(sw.log, "LoadState-GetNbrCpp"):
AdjLists = LoadState(sw)
if AdjLists:
# state is available, process requests for neighbors
sw.log.debug('[%s] state available, length %d' %
(sw.GetName(), AdjLists.Len()))
for item in msglist:
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
msg = Snap.TIntV(FIn)
GetNeighbors(sw, AdjLists, msg)
return
# state not found, initialize it with neighbors
sw.log.debug('[%s] adjlist not found, initializing' % sw.GetName())
Edges = Snap.TIntIntVV()
for item in msglist:
name = sw.GetMsgName(item)
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntIntVV(FIn)
Snap.AddVec64(Edges, Vec)
# first iteration: input are edges, save the state
AdjLists = GetEdges(sw, Edges)
sw.log.debug('[%s] saving adjlist of size %d now' %
(sw.GetName(), AdjLists.Len()))
with perf.Timer(sw.log, "SaveState-GetNbrCpp"):
SaveState(sw, AdjLists)
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "targets"
dmsgout["body"] = {}
sw.Send(0, dmsgout, "2")
def GetNbr(sw):
"""
provide graph neighbors
"""
taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.flog.write("msglist " + str(msglist) + "\n")
sw.flog.flush()
AdjLists = LoadState()
if AdjLists:
# state is available, process requests for neighbors
for item in msglist:
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
msg = Snap.TIntV(FIn)
GetNeighbors(sw, AdjLists, msg)
return
# state not found, initialize it with neighbors
Edges = Snap.TIntV()
for item in msglist:
name = sw.GetMsgName(item)
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
Edges.AddV(Vec)
# first iteration: input are edges, save the state
AdjLists = GetEdges(Edges)
sw.flog.write("state " + str(AdjLists.Len()) + "\n")
sw.flog.flush()
SaveState(AdjLists)
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "targets"
dmsgout["body"] = {}
sw.Send(0, dmsgout, "2")
def GetNbr(sw):
"""
provide graph neighbors
"""
# taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.log.debug("msglist %s" % msglist)
with perf.Timer(sw.log, "LoadState-GetNbrCpp"):
AdjLists = LoadState(sw)
if AdjLists:
# state is available, process requests for neighbors
for item in msglist:
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
msg = Snap.TIntV(FIn)
GetNeighbors(sw, AdjLists, msg)
return
# state not found, initialize it with neighbors
Edges = Snap.TIntV()
for item in msglist:
name = sw.GetMsgName(item)
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
Edges.AddV(Vec)
# first iteration: input are edges, save the state
AdjLists = GetEdges(sw, Edges)
sw.log.debug("state: %d" % AdjLists.Len())
with perf.Timer(sw.log, "SaveState-GetNbrCpp"):
SaveState(sw, AdjLists)
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "targets"
dmsgout["body"] = {}
sw.Send(0, dmsgout, "2")
def LoadState():
fname = sw.GetStateName()
if not os.path.exists(fname):
return None
FIn = Snap.TFIn(Snap.TStr(fname))
AdjLists = Snap.TIntIntVH(FIn)
return AdjLists
def GenGraph(sw):
"""
generate the graph edges
"""
# extract the stubs from the args
# iterate through the input queue and add new items to the stub list
# taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.log.debug("msglist: %s" % msglist)
Stubs = Snap.TIntV() # Stubs is an empty vector
for item in msglist:
# 1) Get item in msglist
# 2) Get name of item
name = sw.GetMsgName(item)
# 3) Get vector associated with name
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
# 4) Add vector to Stubs
Stubs.AddV(Vec)
# 5) Got all stubs, which is of length msglist
# # Randomize the items (aka shuffle)
# Snap.Randomize(Stubs)
#
# # nodes in each task and the number of tasks
# tsize = sw.GetRange()
# ntasks = int(sw.GetVar("gen_tasks"))
#
# # get edges for a specific task
# Tasks = Snap.TIntIntVV(ntasks) # vector of length ntasks containing vectors
# Snap.AssignEdges(Stubs, Tasks, tsize)
ntasks = int(sw.GetVar("gen_tasks"))
seg_bits = int(sw.GetVar('seg_bits'))
tsize = sw.GetRange()
Tasks = Snap.TIntVVV(ntasks)
Stubs = Snap.segment(Stubs, seg_bits) # segmentize stubs
# do segmented random edge assignment
Snap.AssignRandomEdges64(Stubs, Tasks, tsize, seg_bits)
# desegment results
Tasks = Snap.desegmentRandomizedEdges(Tasks, seg_bits, tsize)
# send messages
for i in xrange(0, Tasks.Len()):
sw.log.debug("sending task: %d, len: %d" % (i, Tasks.GetVal(i).Len()))
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def GenGraph(sw):
"""
generate the graph edges
"""
# extract the stubs from the args
# iterate through the input queue and add new items to the stub list
# taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.log.debug("msglist: %s" % msglist)
Stubs = Snap.TIntIntVV() # Stubs is an empty vector
for item in msglist:
# 1) Get item in msglist
# 2) Get name of item
name = sw.GetMsgName(item)
# 3) Get vector associated with name
FIn = Snap.TFIn(Snap.TStr(name))
Vec64 = Snap.TIntIntVV(FIn)
# 4) Add vector to Stubs
#Stubs.AddV(Vec)
Snap.AddVec64(Stubs, Vec64)
# 5) Got all stubs, which is of length msglist
# nodes in each task
tsize = sw.GetRange()
# number of bits in our segment (so seg size is (1<<seg_bits))
seg_bits = int(sw.GetVar('seg_bits'))
# number of tasks
ntasks = int(sw.GetVar("gen_tasks"))
# get edges for a specific task
Tasks = Snap.TIntVVV(ntasks) # vector of length ntasks containing vectors
sw.log.debug('[%s] about to assign random edges' % sw.GetName())
# handles shuffling and random assignment of edges
Snap.AssignRandomEdges64(Stubs, Tasks, tsize, seg_bits)
sw.log.debug('[%s] done assigning random edges' % sw.GetName())
# send messages
for i in xrange(0, Tasks.Len()):
sw.log.debug(LazyStr(lambda: '[%s] sending TIntIntVV of memory size %d to %d' % \
(sw.GetName(), Snap.GetMemSize64(Tasks.GetVal(i)), i)))
sw.Send(i, Tasks.GetVal(i), swsnap=True)
def GenRndGnm():
Graph = snap.GenRndGnm_PNGraph(1000,10000)
print "Graph", str(type(Graph)), Graph.GetNodes(), Graph.GetEdges()
# save the graph
FName = "test2.graph"
print "Save", FName
FOut = snap.TFOut(snap.TStr(FName))
Graph.Save(FOut)
FOut.Flush()
# load the graph
print "Read", FName
FIn = snap.TFIn(snap.TStr(FName))
#Graph2 = snap.TNGraph(FIn)
#Graph2 = snap.TNGraph.Load(FIn)
Graph2 = snap.PNGraph.New()
print "Graph2", str(type(Graph2))
print str(dir(Graph2))
Graph2.Load(FIn)
def SaveState(ds):
fname = sw.GetStateName()
Start = Snap.TInt(ds["start"])
Dist = Snap.TInt(ds["dist"])
Visited = ds["visit"]
FOut = Snap.TFOut(Snap.TStr(fname))
Start.Save(FOut)
Dist.Save(FOut)
Visited.Save(FOut)
FOut.Flush()
def SaveState(sw, ds):
fname = sw.GetStateName()
Start = Snap.TInt(ds["start"])
Dist = Snap.TInt(ds["dist"])
Visited = ds["visit"]
FOut = Snap.TFOut(Snap.TStr(fname))
sw.cum_timer.cum_start("disk")
Start.Save(FOut)
Dist.Save(FOut)
Visited.Save(FOut)
FOut.Flush()
sw.cum_timer.cum_stop("disk")
def LoadState():
fname = sw.GetStateName()
if not os.path.exists(fname):
return None
FIn = Snap.TFIn(Snap.TStr(fname))
Start = Snap.TInt(FIn)
Dist = Snap.TInt(FIn)
Visited = Snap.TIntH(FIn)
ds = {}
ds["start"] = Start.Val
ds["dist"] = Dist.Val
ds["visit"] = Visited
return ds
def LoadState(sw):
fname = sw.GetStateName()
if not os.path.exists(fname):
return None
sw.cum_timer.cum_start("disk")
FIn = Snap.TFIn(Snap.TStr(fname))
Start = Snap.TInt(FIn)
Dist = Snap.TInt(FIn)
Visited = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
ds = {}
ds["start"] = Start.Val
ds["dist"] = Dist.Val
ds["visit"] = Visited
return ds
def SaveState(ds):
fname = sw.GetStateName()
First = Snap.TInt(ds["first"])
Range = Snap.TInt(ds["range"])
Count = Snap.TInt(ds["count"])
Dist = Snap.TInt(ds["dist"])
Start = Snap.TInt(ds["start"])
Visited = ds["visit"]
FOut = Snap.TFOut(Snap.TStr(fname))
First.Save(FOut)
Range.Save(FOut)
Count.Save(FOut)
Dist.Save(FOut)
Start.Save(FOut)
Visited.Save(FOut)
FOut.Flush()
def LoadState():
fname = sw.GetStateName()
if not os.path.exists(fname):
return None
FIn = Snap.TFIn(Snap.TStr(fname))
First = Snap.TInt(FIn)
Range = Snap.TInt(FIn)
Count = Snap.TInt(FIn)
Dist = Snap.TInt(FIn)
Start = Snap.TInt(FIn)
Visited = Snap.TIntV(FIn)
ds = {}
ds["first"] = First.Val
ds["range"] = Range.Val
ds["count"] = Count.Val
ds["dist"] = Dist.Val
ds["start"] = Start.Val
ds["visit"] = Visited
return ds
def Send(self, dstid, d, channel="1", swsnap=False):
#dstnum = dstid / self.range
#dstname = self.target + "-" + str(dstnum)
dstname = self.target[channel] + "-" + str(dstid)
dsthostid = self.tasks.get(dstname)
dshost = self.hosts.get(dsthostid)
self.log.debug('sending to %s (had id %d)' % (dshost, dstid))
if self.local:
fname = self.GetOutName(dstname)
if swsnap:
if not gotsnap:
self.log.error("Snap module is not available")
sys.exit(2)
# Snap vector
if self.local:
FOut = Snap.TFOut(Snap.TStr(fname))
d.Save(FOut)
FOut.Flush()
#print "send Snap task %s, host %s, *** Error: local 'Send' not yet implemented" % (dstname, dshost)
return
client.messagevec(dshost, self.taskname, dstname, d)
return
else:
# json dict
s = json.dumps(d)
# print "send task %s, host %s, msg %s" % (dstname, dshost, s)
if self.local:
f = open(fname, "w")
f.write(s)
f.close()
return
client.message(dshost, self.taskname, dstname, s)
def GenGraph(sw):
"""
generate the graph edges
"""
# extract the stubs from the args
# iterate through the input queue and add new items to the stub list
taskname = sw.GetName()
msglist = sw.GetMsgList()
sw.flog.write("msglist " + str(msglist) + "\n")
sw.flog.flush()
Stubs = Snap.TIntV()
for item in msglist:
sw.flog.write("1 got item " + item + "\n")
sw.flog.flush()
name = sw.GetMsgName(item)
sw.flog.write("2 got name " + name + "\n")
sw.flog.flush()
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.flog.write("3 got vector %d" % (Vec.Len()) + "\n")
sw.flog.flush()
Stubs.AddV(Vec)
#for i in range(0,Vec.Len()):
# stubs.append(Vec.GetVal(i).Val)
#sw.flog.write("4 got stubs %d" % (len(stubs)) + "\n")
sw.flog.write("4 got stubs %d" % (Stubs.Len()) + "\n")
sw.flog.flush()
sw.flog.write("5 got all stubs\n")
sw.flog.flush()
#print taskname,stubs
# randomize the items
Snap.Randomize(Stubs)
#random.shuffle(stubs)
#print taskname + "-r",stubs
# get the pairs
#pairs = zip(stubs[::2], stubs[1::2])
#print taskname,pairs
# nodes in each task and the number of tasks
tsize = sw.GetRange()
ntasks = int(sw.GetVar("gen_tasks"))
# get edges for a specific task
Tasks = Snap.TIntIntVV(ntasks)
Snap.AssignEdges(Stubs, Tasks, tsize)
#print taskname,edges
# send messages
for i in range(0, Tasks.Len()):
#sw.flog.write("sending task %d" % (i) + "\n")
sw.flog.write("sending task %d, len %d" % (i, Tasks.GetVal(i).Len()) +
"\n")
sw.flog.flush()
sw.Send(i, Tasks.GetVal(i), swsnap=True)
import os
import sys
import time
sys.path.append("/home/rok/git/rok/snapworld")
import snap as Snap
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: " + sys.argv[0] + " <file>"
sys.exit(1)
fname = sys.argv[1]
FIn = Snap.TFIn(Snap.TStr(fname))
Vec = Snap.TIntV(FIn)
print "len", Vec.Len()
Vec.Sort()
for i in range(0, Vec.Len()):
print "Vec", i, Vec.GetVal(i).Val
def ManipulateNodesEdges():
'''
Test node, edge creation
'''
NNodes = 10000
NEdges = 100000
FName = "test.graph"
Graph = snap.TUNGraph()
t = Graph.Empty()
# create the nodes
for i in range(0, NNodes):
Graph.AddNode(i)
t = Graph.Empty()
n = Graph.GetNodes()
# create random edges
NCount = NEdges
while NCount > 0:
x = int(random.random() * NNodes)
y = int(random.random() * NNodes)
# skip the loops in this test
if x != y and not Graph.IsEdge(x, y):
n = Graph.AddEdge(x, y)
NCount -= 1
PrintGStats("ManipulateNodesEdges:Graph1", Graph)
# get all the nodes
NCount = 0
NI = Graph.BegNI()
while NI < Graph.EndNI():
NCount += 1
NI.Next()
# get all the edges for all the nodes
ECount1 = 0
NI = Graph.BegNI()
while NI < Graph.EndNI():
ECount1 += NI.GetOutDeg()
NI.Next()
ECount1 = ECount1 / 2
# get all the edges directly
ECount2 = 0
EI = Graph.BegEI()
while EI < Graph.EndEI():
ECount2 += 1
EI.Next()
print "graph ManipulateNodesEdges:Graph2, nodes %d, edges1 %d, edges2 %d" % (
NCount, ECount1, ECount2)
# assignment
Graph1 = Graph
PrintGStats("ManipulateNodesEdges:Graph3", Graph1)
# save the graph
FOut = snap.TFOut(snap.TStr(FName))
Graph.Save(FOut)
FOut.Flush()
# load the graph
FIn = snap.TFIn(snap.TStr(FName))
Graph2 = snap.TUNGraph(FIn)
PrintGStats("ManipulateNodesEdges:Graph4", Graph2)
# remove all the nodes and edges
for i in range(0, NNodes):
n = Graph.GetRndNId()
Graph.DelNode(n)
PrintGStats("ManipulateNodesEdges:Graph5", Graph)
Graph1.Clr()
PrintGStats("ManipulateNodesEdges:Graph6", Graph1)
def ReadVec(fname):
FIn = Snap.TFIn(Snap.TStr(fname))
Vec = Snap.TIntV(FIn)
return Vec
def SaveState(AdjLists):
fname = sw.GetStateName()
FOut = Snap.TFOut(Snap.TStr(fname))
AdjLists.Save(FOut)
FOut.Flush()
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# print "len", Vec.Len()
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
# done, no new nodes
if NewNodes.Len() <= 0:
Visited.GetVal(ds["start"]).Val = 0 # reset start node to 0
timer.start("dist-get-distribution")
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited,DistCount)
l = []
for i in xrange(0, DistCount.Len()):
if DistCount.GetVal(i).Val <= 0:
break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0,dmsgout,"2")
sw.cum_timer.cum_stop("network")
sw.log.info("final: %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances: %s" % str(l))
timer.stop("dist-get-distribution")
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
timer.start("dist-send-all")
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.cum_timer.cum_start("network")
sw.Send(i,Vec1,swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1 # this should get overwritten if we have messages.
# logger gives a warning if that doesn't happen
Visited = ds["visit"]
seg_bits = int(sw.GetVar('seg_bits'))
this_segment_start = Snap.zeroLowOrderBits(ds['first'], seg_bits)
sw.log.debug('this task starts at node %d' % ds['first'])
sw.log.debug('this segment starts at node %d' % this_segment_start)
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV(
) # TODO (smacke): I think this is fine non-segmented
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
FringeSubset = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# it's okay to reassign and then use this later outside of the loop
# since BSP should guarantee that this is the same at every loop iteration
distance = FringeSubset.Last(
).Val + 1 # last value has prev distance, so we inc by 1
FringeSubset.DelLast()
# subtract the starting index
sw.log.debug('[%s] offsetting by first node id' % sw.GetName())
Snap.IncVal(FringeSubset, -(ds["first"] - this_segment_start))
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
# NewNodes will each have the segmented bits zero'd out, as well as
# the high-order bits for this task
sw.log.debug('[%s] calling GetNewNodes1' % sw.GetName())
Snap.GetNewNodes1(FringeSubset, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
ds["dist"] = distance
# This should never be -1 after processing message
if distance == -1:
sw.log.warn("[%s] thinks that the current distance is -1, \
this almost certainly means that things are broken!" % sw.GetName())
nnodes = ds["range"]
ds["count"] += NewNodes.Len() # no. of new things we visited
sw.log.info("distance: %d, new: %d, count: %d, nodes: %d" % \
(distance, NewNodes.Len(), ds["count"], nnodes))
sw.log.debug("testing: %d %d %d" % \
(ds["count"], nnodes, ds["count"] >= nnodes))
# done, no new nodes
if ds["count"] >= nnodes:
sw.log.debug("sending finish output")
if ds["source"] >= 0:
# reset start node to 0
Visited.GetVal(ds["source"] -
ds["first"]).Val = 0 # SMACKE: should be ok
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
# get the maximum positive distance
maxdist = DistCount.Len() - 1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.log.debug("maxdist: %d" % maxdist)
# collect the distances
l = []
for i in xrange(maxdist):
# if DistCount.GetVal(i).Val <= 0: break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["source"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0, dmsgout, "2")
sw.cum_timer.cum_stop("network")
sw.log.debug("final: %s %s" % (str(ds["source"]), str(distance)))
sw.log.debug("distances: %s" % str(l))
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
#ntasks = int(sw.GetVar("gen_tasks"))
ntasks = (1 << seg_bits
) / tsize # reduce number to only tasks within same segment
Tasks = Snap.TIntIntVV(ntasks)
# we will only ever send to tasks in the same segment, but # TODO (smacke) not wasting space anymore
# the wasted space shouldn't hurt that much
sw.log.debug('[%s] increase nodes to within-segment values now' %
sw.GetName())
Snap.IncVal(NewNodes, ds["first"] - this_segment_start)
# assign nodes to tasks
sw.log.debug('[%s] calling Nodes2Tasks1' % sw.GetName())
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
# All of the GetNbr tasks are in the same segment as this task
# so this should still work; we just have to find the base task for
# this segment and add it to all of the task indexes in Tasks
timer.stop("dist-collect-nodes")
# send the messages
timer.start("dist-send-all")
for i in xrange(Tasks.Len()):
Vec1 = Tasks.GetVal(i)
sw.log.debug('Vec1 length: %d' % Vec1.Len())
if Vec1.Len() <= 0:
continue
# add task# at the end # TODO (smacke): I still don't understand this terminology
Vec1.Add(distance)
sw.cum_timer.cum_start("network")
# we need to send to the correct segment, from which our
# tasks are offset
sw.Send(i + this_segment_start / tsize, Vec1, swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1
Visited = ds["visit"]
#print "Visited", type(Visited)
# nodes to add are on the input
NewNodes = Snap.TIntV()
t1 = time.time()
for item in msglist:
name = sw.GetMsgName(item)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f input %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
distance = Vec.Last().Val + 1
Vec.DelLast()
# subtract the starting index
Snap.IncVal(Vec, -ds["first"])
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f reading %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
#print "len", Vec.Len()
# get new nodes, not visited before
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance);
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f compute %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
ds["dist"] = distance
nnodes = ds["range"]
ds["count"] += NewNodes.Len()
sw.flog.write("distance %d, new %d, count %d, nodes %d\n" % (
distance, NewNodes.Len(), ds["count"], nnodes))
sw.flog.flush()
# done, no new nodes
#if NewNodes.Len() <= 0:
sw.flog.write("testing %d %d %d\n" % (ds["count"], nnodes, ds["count"] >= nnodes))
sw.flog.flush()
if ds["count"] >= nnodes:
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#print "%s.%03d output start" % (time.ctime(t2)[11:19], tmsec)
#t1 = t2
sw.flog.write("sending finish output\n")
sw.flog.flush()
if ds["start"] >= 0:
# reset start node to 0
Visited.GetVal(ds["start"]-ds["first"]).Val = 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited,DistCount)
#for i in xrange(0, DistCount.Len()):
# print "dist", i, DistCount.GetVal(i).Val
#for snode in xrange(0,nnodes):
# distance = Visited.GetVal(snode).Val
# if not dcount.has_key(distance):
# dcount[distance] = 0
# dcount[distance] += 1
# get the maximum positive distance
maxdist = DistCount.Len()-1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.flog.write("maxdist %d\n" % (maxdist))
sw.flog.flush()
# collect the distances
l = []
for i in xrange(0, maxdist):
#if DistCount.GetVal(i).Val <= 0:
# break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.Send(0,dmsgout,"2")
sw.flog.write("final %s %s\n" % (str(ds["start"]), str(distance)))
sw.flog.write("distances " + str(l) + "\n")
sw.flog.flush()
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f output done" % (
# time.ctime(t2)[11:19], tmsec, tdiff)
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
Snap.IncVal(NewNodes, ds["first"])
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
#for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(distance)
sw.Send(i,Vec1,swsnap=True)
# printstat.py
# python printstat.py data/p2p-Gnutella08.txt p2p
import os
import sys
sys.path.append("../swig")
import snap
if __name__ == '__main__':
if len(sys.argv) < 3:
print "Usage: " + sys.argv[0] + " <graph_file> <header_text>"
sys.exit(1)
gname = sys.argv[1]
header = sys.argv[2]
tname = snap.TStr(gname)
tlabel = snap.TStr(os.path.splitext(gname)[0])
theader = snap.TStr(header)
g = snap.LoadEdgeList(tname, 0, 1)
snap.PrintGraphStatTable(g, tlabel, theader)
import os
import sys
import time
sys.path.append("/home/rok/git/rok/snapworld")
import snap
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: " + sys.argv[0] + " <file>"
sys.exit(1)
fname = sys.argv[1]
FIn = snap.TFIn(snap.TStr(fname))
Vec = snap.TIntV(FIn)
print "len", Vec.Len()
Vec.Sort()
for i in range(0, Vec.Len()):
print "Vec", i, Vec.GetVal(i).Val
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
# print "Visited", type(Visited)
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntH()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# print "input", name
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
# print "len", Vec.Len()
# get new nodes, not visited before
timer.start("dist-nodes-iter")
Snap.GetNewNodes(Vec, Visited, NewNodes, distance)
timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
# done, no new nodes
if NewNodes.Len() <= 0:
timer.start("dist-get-distribution")
# get distance distribution
dcount = {}
# TODO move this loop to SNAP C++
VIter = Visited.BegI()
while not VIter.IsEnd():
# snode = VIter.GetKey().Val
distance = VIter.GetDat().Val
if not dcount.has_key(distance):
dcount[distance] = 0
dcount[distance] += 1
VIter.Next()
nnodes = int(sw.GetVar("nodes"))
l = []
for i in xrange(0, nnodes):
if not dcount.has_key(i):
break
l.append(dcount[i])
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0,dmsgout,"2")
sw.cum_timer.cum_stop("network")
sw.log.info("final %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances %s" % str(l))
timer.stop("dist-get-distribution")
return
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
timer.start("dist-send-all")
for i in range(0,Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.cum_timer.cum_start("network")
sw.Send(i,Vec1,swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")
def AddNewNodes(taskindex, sw, ds, msglist):
ds["dist"] += 1
distance = ds["dist"]
Visited = ds["visit"]
#print "Visited", type(Visited)
# nodes to add are on the input
NewNodes = Snap.TIntV()
t1 = time.time()
for item in msglist:
name = sw.GetMsgName(item)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f input %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f reading %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
#print "len", Vec.Len()
# get new nodes, not visited before
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f compute %s" % (
# time.ctime(t2)[11:19], tmsec, tdiff, name)
#t1 = t2
nnodes = int(sw.GetVar("nodes"))
#ds["count"] += NewNodes.Len()
# done, no new nodes
#if ds["count"] >= nnodes:
if NewNodes.Len() <= 0:
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#print "%s.%03d output start" % (time.ctime(t2)[11:19], tmsec)
#t1 = t2
Visited.GetVal(ds["start"]).Val = 0 # reset start node to 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
#for i in xrange(0, DistCount.Len()):
# print "dist", i, DistCount.GetVal(i).Val
#for snode in xrange(0,nnodes):
# distance = Visited.GetVal(snode).Val
# if not dcount.has_key(distance):
# dcount[distance] = 0
# dcount[distance] += 1
l = []
for i in xrange(0, DistCount.Len()):
if DistCount.GetVal(i).Val <= 0:
break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.Send(0, dmsgout, "2")
sw.flog.write("final %s %s\n" % (str(ds["start"]), str(distance)))
sw.flog.write("distances " + str(l) + "\n")
sw.flog.flush()
#t2 = time.time()
#tmsec = int(t2*1000) % 1000
#tdiff = (t2 - t1)
#print "%s.%03d %.3f output done" % (
# time.ctime(t2)[11:19], tmsec, tdiff)
return
# nodes in each task
tsize = sw.GetRange()
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
#for i in range(0,Tasks.Len()):
# print "sending task %d, len %d" % (i, Tasks.GetVal(i).Len())
# send the messages
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(taskindex)
sw.Send(i, Vec1, swsnap=True)
import snap
for item in dir(snap):
if item.find("Load") >= 0:
print item
g = snap.LoadEdgeList_PNGraph(snap.TStr("soc-Epinions1.txt"), 0, 1)
print dir(g)
print g.GetNodes()
print g.GetEdges()
def run_tests(num_iterations=3, min_nodes_exponent=3, max_nodes_exponent=4):
'''
Perform tests with specified exponent range
'''
if verbose:
print "Running results from %e to %e" % (min_nodes_exponent,
max_nodes_exponent)
Rnd = snap.TRnd()
for exp in range(min_nodes_exponent, max_nodes_exponent + 1):
for n in range(num_iterations):
if verbose:
print "Iteration: %d of %d" % (n + 1, num_iterations)
# Random number of nodes of degree i
NNodes = 10**exp
for avg_deg in range(min_degree_edges, max_degree_edges + 1):
for g in graph_types:
if deterministic:
if verbose:
print "Deterministic mode, putting seed"
else:
if verbose:
print "Non-deterministic mode"
Rnd.PutSeed(0)
if verbose: print "Using average degree of 10^%d" % avg_deg
NEdges = NNodes * (10**avg_deg)
Graph = None
if g in [
'rmat', 'rand_ngraph', 'syn_ngraph', 'syn_negraph'
]:
Type = "directed"
elif g in ['sw', 'pref', 'rand_ungraph']:
Type = "undirected"
elif g in ['rand_neanet', 'syn_neanet']:
Type = "attribute"
else:
print "Unknown graph type: %s" % g
sys.exit(1)
StartTime = clock()
FName = os.path.join(
RESULTS_DIR, "%s_10e%d_deg%d_%d.graph" %
(g, exp, NEdges / NNodes, n))
if not generate:
if os.path.exists(FName):
try:
if verbose:
print "Loading '%s' from ...'%s'" % (
g, FName),
sys.stdout.flush()
FIn = snap.TFIn(snap.TStr(FName))
if Type == "directed":
Graph = snap.PNGraph_New()
elif Type == "undirected":
Graph = snap.PUNGraph_New()
elif Type == "attribute":
Graph = snap.PNEANet_New()
Graph = Graph.Load(FIn)
if verbose: print "done"
if verbose:
print "Re-loaded graph with %d Nodes and %d Edges" % \
(Graph.GetNodes(), Graph.GetEdges())
except Exception, e:
print "Unable to load graph file, '%s': %s" % (
FName, str(e))
# else:
# print "File not found: %s" % FName
if not Graph:
try:
# User wants to re-generate graph, or no graph data available.
if verbose:
print "Generating '%s %s' graph with %e nodes, %e edges..." % \
(Type, g, NNodes, NEdges),
sys.stdout.flush()
Graph = generate_graph(NNodes, NEdges, g, Type,
Rnd)
if verbose: print "done"
if opt_write:
# Save the graph
if verbose:
print "Saving '%s' graph to file '%s'..." % (
g, FName),
sys.stdout.flush()
if Graph:
FOut = snap.TFOut(snap.TStr(FName))
Graph.__ref__().Save(
FOut) # Save as TUNGraph or TNGraph
FOut.Flush()
if verbose: print "done"
except Exception, e:
print "Unable to generate/save graph file, '%s': %s" % \
(FName, str(e))
continue
TimeGenerate = clock() - StartTime
print "Running tests...",
sys.stdout.flush()
StartTime = clock()
if Type == 'directed':
results = benchmark_ngraph(Graph)
elif Type == 'undirected':
results = benchmark_ungraph(Graph)
elif Type == 'attribute':
results = benchmark_neanet(Graph)
if verbose: print "done"
TimeElapsed = clock() - StartTime
print "Elapsed Time = %.4f sec" % TimeElapsed
row_header = [
"Hostname", "Model", "Type", "Nodes", "Edges",
"StartTime", "Generation Time", "Run Time"
]
print "Header: %s" % " ".join(row_header)
import csv
with open(results_file, 'a+') as csvfile:
writer = csv.writer(csvfile)
if verbose:
print "Writing to '%s'..." % results_file,
sys.stdout.flush()
row = [
HOSTNAME, g, Type, NNodes, NEdges,
datetime.now().strftime("%d/%b/%Y:%H:%M:%S"),
TimeGenerate, TimeElapsed
]
if verbose: print "done"
print "Time Data: %s" % repr(row)
writer.writerow(row)
print "-" * 75
def AddNewNodes(taskindex, sw, ds, msglist):
# all the nodes were visited already
if ds["count"] >= ds["range"]:
return
distance = -1
Visited = ds["visit"]
timer = perf.Timer(sw.log)
# nodes to add are on the input
NewNodes = Snap.TIntV()
timer.start("dist-msglist-iter")
perf.DirSize(sw.log, sw.qin, "GetDist-qin")
for item in msglist:
sw.cum_timer.cum_start("disk")
name = sw.GetMsgName(item)
# read the input nodes
FIn = Snap.TFIn(Snap.TStr(name))
Vec = Snap.TIntV(FIn)
sw.cum_timer.cum_stop("disk")
distance = Vec.Last(
).Val + 1 # update current distance for fringe nodes
Vec.DelLast()
# subtract the starting index
Snap.IncVal(Vec, -ds["first"])
# print "len", Vec.Len()
# get new nodes, not visited before
# timer.start("dist-nodes-iter")
Snap.GetNewNodes1(Vec, Visited, NewNodes, distance)
# timer.stop("dist-nodes-iter")
timer.stop("dist-msglist-iter")
ds["dist"] = distance
nnodes = ds["range"]
ds["count"] += NewNodes.Len()
sw.log.info("distance: %d, new: %d, count: %d, nodes: %d" % \
(distance, NewNodes.Len(), ds["count"], nnodes))
# done, no new nodes
sw.log.debug("testing: %d %d %d" % \
(ds["count"], nnodes, ds["count"] >= nnodes))
if ds["count"] >= nnodes:
sw.log.info("sending finish output")
if ds["start"] >= 0:
# reset start node to 0
Visited.GetVal(ds["start"] - ds["first"]).Val = 0
# get distance distribution, assume 1000 is the max
DistCount = Snap.TIntV(1000)
Snap.ZeroVec(DistCount)
Snap.GetDistances(Visited, DistCount)
# get the maximum positive distance
maxdist = DistCount.Len() - 1
while (maxdist > 0) and (DistCount.GetVal(maxdist).Val == 0):
maxdist -= 1
maxdist += 1
sw.log.info("maxdist: %d" % maxdist)
# collect the distances
l = []
for i in xrange(maxdist):
# if DistCount.GetVal(i).Val <= 0: break
l.append(DistCount.GetVal(i).Val)
dmsg = {}
dmsg["start"] = ds["start"]
dmsg["dist"] = l
dmsgout = {}
dmsgout["src"] = sw.GetName()
dmsgout["cmd"] = "results"
dmsgout["body"] = dmsg
sw.cum_timer.cum_start("network")
sw.Send(0, dmsgout, "2")
sw.cum_timer.cum_stop("network")
sw.log.info("final: %s %s" % (str(ds["start"]), str(distance)))
sw.log.info("distances: %s" % str(l))
# nodes in each task
tsize = sw.GetRange()
timer.start("dist-collect-nodes")
# collect nodes for the same task
ntasks = int(sw.GetVar("gen_tasks"))
Tasks = Snap.TIntIntVV(ntasks)
Snap.IncVal(NewNodes, ds["first"])
# assign nodes to tasks
Snap.Nodes2Tasks1(NewNodes, Tasks, tsize)
timer.stop("dist-collect-nodes")
# send the messages
timer.start("dist-send-all")
for i in range(0, Tasks.Len()):
Vec1 = Tasks.GetVal(i)
if Vec1.Len() <= 0:
continue
# add task# at the end
Vec1.Add(distance)
sw.cum_timer.cum_start("network")
sw.Send(i, Vec1, swsnap=True)
sw.cum_timer.cum_stop("network")
timer.stop("dist-send-all")