def GenGraph(tname,args): """ generate the graph edges args, arguments as a string """ step = int(tname.split("-")[1]) # extract the stubs from the args # iterate through the input queue and add new items to the stub list stubs = [] for item in args: l = simplejson.loads(item) stubs.extend(l) #print stubs print tname,stubs # randomize the items random.shuffle(stubs) #print tname + "-r",stubs # get the pairs pairs = zip(stubs[::2], stubs[1::2]) print tname,pairs # distribute the stubs randomly to the tasks tsize = comm.mgetconfig("tsize") # get edges for a specific task edges = {} for pair in pairs: esrc = pair[0] edst = pair[1] # add the edge twice for both directions tdst = TaskId(esrc, tsize) if not edges.has_key(tdst): edges[tdst] = [] l = [esrc, edst] edges[tdst].append(l) tdst = TaskId(edst, tsize) if not edges.has_key(tdst): edges[tdst] = [] l = [edst, esrc] edges[tdst].append(l) print tname,edges for key, value in edges.iteritems(): tdst = "E%s" % (str(key)) + "-" + str(step+1) targs = simplejson.dumps(value) print tdst,targs comm.mroute(tname,tdst,targs)
def GenGraph(tname, args): """ generate the graph edges args, arguments as a string """ step = int(tname.split("-")[1]) # extract the stubs from the args # iterate through the input queue and add new items to the stub list stubs = [] for item in args: l = simplejson.loads(item) stubs.extend(l) #print stubs print tname, stubs # randomize the items random.shuffle(stubs) #print tname + "-r",stubs # get the pairs pairs = zip(stubs[::2], stubs[1::2]) print tname, pairs # distribute the stubs randomly to the tasks tsize = comm.mgetconfig("tsize") # get edges for a specific task edges = {} for pair in pairs: esrc = pair[0] edst = pair[1] # add the edge twice for both directions tdst = TaskId(esrc, tsize) if not edges.has_key(tdst): edges[tdst] = [] l = [esrc, edst] edges[tdst].append(l) tdst = TaskId(edst, tsize) if not edges.has_key(tdst): edges[tdst] = [] l = [edst, esrc] edges[tdst].append(l) print tname, edges for key, value in edges.iteritems(): tdst = "E%s" % (str(key)) + "-" + str(step + 1) targs = simplejson.dumps(value) print tdst, targs comm.mroute(tname, tdst, targs)
def GenStubs(tname,args): """ determine degrees for all the nodes, generate the stubs and distribute them args, arguments as a string """ step = int(tname.split("-")[1]) argwords = args[0] largs = argwords.split("\t") #print largs ns = int(largs[0]) ne = int(largs[1]) dis = largs[2] print "*task* %s %d %d %s" % (tname, ns, ne, dis) # determine node degrees i = ns ddeg = {} while i <= ne: #deg = StdDist(distmean,distvar) deg = 3 ddeg[i] = deg print "*task* %s, node %s, degree %s" % (tname, str(i), str(deg)) i += 1 print ddeg # distribute the stubs randomly to the tasks ntasks = comm.mgetconfig("tasks") print "__tasks__ %s\t%s" % (tname, str(ntasks)) # one item per task, each task has a list of stubs dstubs = {} for key,value in ddeg.iteritems(): for i in range(0,value): t = int(random.random() * ntasks) if not dstubs.has_key(t): dstubs[t] = [] dstubs[t].append(key) for key, value in dstubs.iteritems(): tdst = "S%s" % (str(key)) + "-" + str(step+1) targs = simplejson.dumps(value) print tdst,targs comm.mroute(tname,tdst,targs)
def GenStubs(tname, args): """ determine degrees for all the nodes, generate the stubs and distribute them args, arguments as a string """ step = int(tname.split("-")[1]) argwords = args[0] largs = argwords.split("\t") #print largs ns = int(largs[0]) ne = int(largs[1]) dis = largs[2] print "*task* %s %d %d %s" % (tname, ns, ne, dis) # determine node degrees i = ns ddeg = {} while i <= ne: #deg = StdDist(distmean,distvar) deg = 3 ddeg[i] = deg print "*task* %s, node %s, degree %s" % (tname, str(i), str(deg)) i += 1 print ddeg # distribute the stubs randomly to the tasks ntasks = comm.mgetconfig("tasks") print "__tasks__ %s\t%s" % (tname, str(ntasks)) # one item per task, each task has a list of stubs dstubs = {} for key, value in ddeg.iteritems(): for i in range(0, value): t = int(random.random() * ntasks) if not dstubs.has_key(t): dstubs[t] = [] dstubs[t].append(key) for key, value in dstubs.iteritems(): tdst = "S%s" % (str(key)) + "-" + str(step + 1) targs = simplejson.dumps(value) print tdst, targs comm.mroute(tname, tdst, targs)
def GetDist(tname,args): """ find the node distance args, arguments as a string """ print "GetDist", tname task = tname.split("-")[0] step = int(tname.split("-")[1]) tsize = comm.mgetconfig("tsize") # get the initial arguments: starting node and its neighbors dinit = simplejson.loads(args[0]) node = dinit["node"] nbrlist = dinit["nbrs"] # no visited nodes yet, first iteration visited = {} distance = 0 print "*distance*", tname, node, distance visited[node] = distance while True: # process the new neighbors distance += 1 newnodes = [] # process all the input elements for arg in args: dinit = simplejson.loads(arg) srcnode = dinit["node"] nbrlist = dinit["nbrs"] for item in nbrlist: # skip nodes already visited if item in visited: continue # add new nodes to the visited nodes and the new nodes print "*distance*", tname, item, distance visited[item] = distance newnodes.append(item) # done, if there are no more new nodes if len(newnodes) <= 0: break # send new visited nodes to the graph nodes to find their neighbors # collect nodes for the same task dtasks = {} for ndst in newnodes: tn = TaskId(ndst,tsize) if not dtasks.has_key(tn): dtasks[tn] = [] dtasks[tn].append(ndst) print "dtasks", dtasks # send the messages step += 1 for tn,args in dtasks.iteritems(): dmsg = {} dmsg["task"] = task dmsg["nodes"] = args tdst = "E%s" % (str(tn)) + "-" + str(step) targs = simplejson.dumps(dmsg) print tdst,targs comm.mroute(tname,tdst,targs) # wait for another iteration yield # move the task name to the next step step += 1 tname = task + "-" + str(step) # get the input queue args = comm.mgetargs(tname) # end of while, repeat the loop print "*finish*", tname # TODO calculate statistics return
def GetDist(tname, args): """ find the node distance args, arguments as a string """ print "GetDist", tname task = tname.split("-")[0] step = int(tname.split("-")[1]) tsize = comm.mgetconfig("tsize") # get the initial arguments: starting node and its neighbors dinit = simplejson.loads(args[0]) node = dinit["node"] nbrlist = dinit["nbrs"] # no visited nodes yet, first iteration visited = {} distance = 0 print "*distance*", tname, node, distance visited[node] = distance while True: # process the new neighbors distance += 1 newnodes = [] # process all the input elements for arg in args: dinit = simplejson.loads(arg) srcnode = dinit["node"] nbrlist = dinit["nbrs"] for item in nbrlist: # skip nodes already visited if item in visited: continue # add new nodes to the visited nodes and the new nodes print "*distance*", tname, item, distance visited[item] = distance newnodes.append(item) # done, if there are no more new nodes if len(newnodes) <= 0: break # send new visited nodes to the graph nodes to find their neighbors # collect nodes for the same task dtasks = {} for ndst in newnodes: tn = TaskId(ndst, tsize) if not dtasks.has_key(tn): dtasks[tn] = [] dtasks[tn].append(ndst) print "dtasks", dtasks # send the messages step += 1 for tn, args in dtasks.iteritems(): dmsg = {} dmsg["task"] = task dmsg["nodes"] = args tdst = "E%s" % (str(tn)) + "-" + str(step) targs = simplejson.dumps(dmsg) print tdst, targs comm.mroute(tname, tdst, targs) # wait for another iteration yield # move the task name to the next step step += 1 tname = task + "-" + str(step) # get the input queue args = comm.mgetargs(tname) # end of while, repeat the loop print "*finish*", tname # TODO calculate statistics return