def Dijkstra_p(G,start,end=None): D = {} # dictionary of final distances P = {} # dictionary of predecessors Q = priorityDictionary() # est.dist. of non-final vert. Q[start] = 0 comm = PSim(len(Q)+1) if comm.rank==0: i = 1 for v in Q: D[v] = Q[v] if v == end: break comm.send(i, v) i = i + 1 else: v = comm.recv(0) for w in G[v]: vwLength = D[v] + G[v][w] if w in D: if vwLength < D[w]: raise ValueError, "Dijkstra: found better path to already-final vertex" elif w not in Q or vwLength < Q[w]: Q[w] = vwLength P[w] = v return (D,P)
def test(): comm=PSim(5,SWITCH) if comm.rank==0: print 'start test' a=sum(comm.all2all_broadcast(comm.rank)) comm.barrier() b=comm.all2all_reduce(comm.rank) if a!=10 or a!=b: print 'from process', comm.rank raise Exception if comm.rank==0: print 'test passed'
def scalar_product_test2(n, p): comm = PSim(p) a = b = None if comm.rank == 0: a = [random.random() for i in range(n)] b = [random.random() for i in range(n)] a = comm.one2all_scatter(0, a) b = comm.one2all_scatter(0, b) scalar = sum(a[i] * b[i] for i in range(len(a))) scalar = comm.all2one_reduce(0, scalar) if comm.rank == 0: print(scalar)
def scalar_product_test2(n,p): import random from psim import PSim comm = PSim(p) a = b = None if comm.rank==0: a = [random.random() for i in range(n)] b = [random.random() for i in range(n)] a = comm.one2all_scatter(0,a) b = comm.one2all_scatter(0,b) scalar = sum(a[i]*b[i] for i in range(len(a))) scalar = comm.all2one_reduce(0,scalar) if comm.rank == 0: print scalar
def Parallel(V, E, w, s): l = V.keys() """ Create a new process for each start vector. Note that there is an "extra" because the root process does NOT handle any work - it just scatters and gathers data. """ comm = PSim(len(l) + 1) # Scatter if comm.rank==0: # Loop through all of the processes for i in range(1, len(l)+1): comm.send(i, l[i-1]) else: curr = comm.recv(0) res = Dijkstra(V, curr) comm.send(0, res) # Gather if comm.rank==0: # Loop through all of the processes for i in range(1, len(l)+1): res = comm.recv(i) print "" print "Start: " + str(l[i-1]) print "Path: " + str(res)
def mergesort_test(n, p): comm = PSim(p) if comm.rank == 0: data = [random.random() for i in range(n)] comm.send(1, data[n/2:]) mergesort(data, 0, n/2) data[n/2:] = comm.recv(1) merge(data, 0, n/2, n) print(data) else: data = comm.recv(0) mergesort(data) comm.send(0, data)
def test(): comm = PSim(5, SWITCH) if comm.rank == 0: print 'start test' a = sum(comm.all2all_broadcast(comm.rank)) comm.barrier() b = comm.all2all_reduce(comm.rank) if a != 10 or a != b: print 'from process', comm.rank raise Exception if comm.rank == 0: print 'test passed'
def mergesort_test(n,p): import random from psim import PSim comm = PSim(p) if comm.rank==0: data = [random.random() for i in range(n)] comm.send(1, data[n/2:]) mergesort(data,0,n/2) data[n/2:] = comm.recv(1) merge(data,0,n/2,n) print data else: data = comm.recv(0) mergesort(data) comm.send(0,data)
def scalar_product_test1(n, p): comm = PSim(p) h = n / p if comm.rank == 0: a = [random.random() for i in range(n)] b = [random.random() for i in range(n)] for k in range(1, p): comm.send(k, a[k * h:k * h + h]) comm.send(k, b[k * h:k * h + h]) else: a = comm.recv(0) b = comm.recv(0) scalar = sum(a[i] * b[i] for i in range(h)) if comm.rank == 0: for k in range(1, p): scalar += comm.recv(k) print(scalar) else: comm.send(0, scalar)
def Dijkstra_pf(graph, start): vertices, links = graph P = [PrimVertex(i, links) for i in vertices] Q = [P[i] for i in vertices if not i == start] vertex = P[start] vertex.closest_dist = 0 comm = PSim(len(vertices) + 1) while Q: i = 1 for neighbor_id, length in vertex.neighbors: if comm.rank == 0: # comm.send(i, (P[neighbor_id], length)) comm.send(i, (neighbor_id, length)) else: # neighbor, length = comm.recv(0) nid, lth = comm.recv(0) neighbor = P[nid] dist = lth + vertex.closest_dist if neighbor in Q and dist < neighbor.closest_dist: neighbor.closest = vertex neighbor.closest_dist = dist comm.send(0, (nid, neighbor)) i = i + 1 if comm.rank == 0: l = [] # Gather the message from each iteration for i in range(1, i): nid, r = comm.recv(i) l.append(r) neighbor = P[nid] if r.closest_dist < neighbor.closest_dist: neighbor.closest = vertex neighbor.closest_dist = r.closest_dist heapify(Q) vertex = heappop(Q) # for v in P: print v.closest return [(v.id, v.closest.id, v.closest_dist) for v in P if not v.id == start] """if comm.rank==0:
def scalar_product_test1(n,p): import random from psim import PSim comm = PSim(p) h = n/p if comm.rank==0: a = [random.random() for i in range(n)] b = [random.random() for i in range(n)] for k in range(1,p): comm.send(k, a[k*h:k*h+h]) comm.send(k, b[k*h:k*h+h]) else: a = comm.recv(0) b = comm.recv(0) scalar = sum(a[i]*b[i] for i in range(h)) if comm.rank == 0: for k in range(1,p): scalar += comm.recv(k) print scalar else: comm.send(0,scalar)
def bb(adjacency, p=1): n = len(adjacency) comm = PSim(p) Q = [] path = [0] Q.append(Vertex(path)) bound = float("inf') optimal = None local_vertices = comm.one2all_scatter(0, range(n)) while True: if comm.rank == 0: vertex = Q.pop() if Q else None else: vertex = None vertex = comm.one2all_broadcast(0, vertex) if vertex is None: break P = [] for k in local_vertices: if not k in vertex.path: new_path = vertex.path+[k] new_path_length = weight(new_path, adjacency) if new_path_length<bound: if len(new_path) == n: new_path.append(new_path[0]) new_path_length = weight(new_path, adjacency) if new_path_length<bound: bound = new_path_length # bcast optimal = new_path # bcast else: new_vertex = Vertex(new_path) P.append(new_vertex) # fix this print(new_path, new_path_length) x = (bound, optimal) x = comm.all2all_reduce(x, lambda a, b: min(a, b)) (bound, optimal) = x P = comm.all2one_collect(0, P) if comm.rank == 0: for item in P: Q+=item return optimal, bound
# Begin program execution by supplying n verticies, indexSize (5 would # mean vertex names would consist of 5 random characters, such as ABCDE; # the higher the number, the more unique vertex names that are available), # maxEdges indicates the maximum number of edges from any vertex, and min/max # cost sets the range of costs for the randomly generated cost data. n = 4096 indexSize = 4 maxEdges = 4 #must be less than n minCost = 1 maxCost = 100 # Set the dimensions and processors for this program execution. d = 4 p = 2**d comm = PSim(p) #Generate the random values including source/destination verticies if comm.rank==0: start = time.time() index,revIndex,graph,distances = genData(n,indexSize,maxEdges,minCost,maxCost) source = revIndex[random.randint(0,len(revIndex)-1)] destination = source while(destination == source): destination = revIndex[random.randint(0,len(revIndex)-1)] else: index,revIndex,graph,distances = None,None,None,None source = None destination = None #Distribute a portion of the data to all nodes
print 'Number of Vertices in the Graph: ', n print 'Number of Edges in the Graph: ', e print 'Use heuristic methods (only applies to Parallel Kruskal MST)?: ', h print 'Run serial Kruskal MST?: ', s #run the serial Kruskal Algorithm if chosen if s == 1: serial_graph = make_graph_tuple(n,e) #start the timer st = time.time() MST = kruskal_serial(n, serial_graph) print MST #stop timer #print 'Total Time: ',(time.time() - st) else: comm = PSim(p) #create nodes, node 0 is the master #master node if comm.rank == 0: #the master node creates the initial graph the_graph = make_graph_tuple(n,e) #start the timer st = time.time() #break up edges and send to other worker nodes to #sort the edges by weight num_edges = e / p #node 0's edges to sort local_edges_to_sort = the_graph[0:num_edges] for i in range (1,p): comm.send(i, the_graph[num_edges*i:(num_edges*i)+num_edges]) #sort own local piece
# Begin program execution by supplying n verticies, indexSize (5 would # mean vertex names would consist of 5 random characters, such as ABCDE; # the higher the number, the more unique vertex names that are available), # maxEdges indicates the maximum number of edges from any vertex, and min/max # cost sets the range of costs for the randomly generated cost data. n = 4096 indexSize = 4 maxEdges = 4 #must be less than n minCost = 1 maxCost = 100 # Set the dimensions and processors for this program execution. d = 4 p = 2**d comm = PSim(p) #Generate the random values including source/destination verticies if comm.rank == 0: start = time.time() index, revIndex, graph, distances = genData(n, indexSize, maxEdges, minCost, maxCost) source = revIndex[random.randint(0, len(revIndex) - 1)] destination = source while (destination == source): destination = revIndex[random.randint(0, len(revIndex) - 1)] else: index, revIndex, graph, distances = None, None, None, None source = None destination = None
def dijkstra_p(g, source, p): """ PYsim parallel implementation of dijkstra's algorithm Result: One source shourtest paths to all nodes Implementation parallelizes finding minimums and updating the results (inner loop is parallelized) Uses the method of collecting of intermediate minimums and broadcasting of newly found path to all processes """ # Initialize communicator comm = PSim(p) myRank = comm.rank chunkSize = len(g) / p # Scatter the graph by partitioning it one dimensionally, # along with known distances to source, and remaining node information myGraphChunk = comm.one2all_scatter(0, g) localResultView = comm.one2all_scatter(0, g[source]) localRemainView = comm.one2all_scatter(0, g[source]) # Each process will iterate q times, where q is number of nodes in graph for q in range(len(g)): # Get shortest path to locally un-visited nodes lowestLocalDist = min(localRemainView) lowestLocalDistIdx = localRemainView.index(lowestLocalDist) # Collect the lowest distances and node indexes from all processes gatheredCandidatesDist = comm.all2one_collect(0, lowestLocalDist) gatheredCandidatesIdxs = comm.all2one_collect(0, lowestLocalDistIdx + myRank * chunkSize) processWithLowestDist = -1 addedNodeIdx = -1 lowestOverallDistance = -1 # One 'main' process finds the global minimum if myRank == 0: lowestOverallDistance = min(gatheredCandidatesDist) processWithLowestDist = gatheredCandidatesDist.index(lowestOverallDistance) addedNodeIdx = gatheredCandidatesIdxs[processWithLowestDist] # 'Main' process broadcasts the finding to all processes receivedBroadcast = comm.one2all_broadcast(0, (processWithLowestDist, addedNodeIdx, lowestOverallDistance)) # If this process is responsible for the newly found shortest path, # then mark this node as visited. if myRank == receivedBroadcast[0]: indexToUpdate = receivedBroadcast[1] - myRank * chunkSize localRemainView[indexToUpdate] = "" # Each process loop over all connections to this last visited node # that they have the visibility to. for iter in range(len(localResultView)): potentialNewDistance = myGraphChunk[iter][receivedBroadcast[1]] + receivedBroadcast[2] # If the distances through this node are less than what we know if potentialNewDistance < localResultView[iter]: # Update the local portion of the result list and # remaining nodes list localResultView[iter] = potentialNewDistance localRemainView[iter] = potentialNewDistance # Reduce the final result to a single list and return it reassembled = comm.all2one_reduce(0, localResultView) if myRank == 0: return reassembled else: return []
def dijkstra_p(g, source, p): """ PYsim parallel implementation of dijkstra's algorithm Result: One source shourtest paths to all nodes Implementation parallelizes finding minimums and updating the results (inner loop is parallelized) Uses the method of collecting of intermediate minimums and broadcasting of newly found path to all processes """ # Initialize communicator comm = PSim(p) myRank = comm.rank chunkSize = len(g) / p # Scatter the graph by partitioning it one dimensionally, # along with known distances to source, and remaining node information myGraphChunk = comm.one2all_scatter(0, g) localResultView = comm.one2all_scatter(0, g[source]) localRemainView = comm.one2all_scatter(0, g[source]) # Each process will iterate q times, where q is number of nodes in graph for q in range(len(g)): # Get shortest path to locally un-visited nodes lowestLocalDist = min(localRemainView) lowestLocalDistIdx = localRemainView.index(lowestLocalDist) # Collect the lowest distances and node indexes from all processes gatheredCandidatesDist = comm.all2one_collect(0, lowestLocalDist) gatheredCandidatesIdxs = comm.all2one_collect( 0, lowestLocalDistIdx + myRank * chunkSize) processWithLowestDist = -1 addedNodeIdx = -1 lowestOverallDistance = -1 # One 'main' process finds the global minimum if myRank == 0: lowestOverallDistance = min(gatheredCandidatesDist) processWithLowestDist = gatheredCandidatesDist.index( lowestOverallDistance) addedNodeIdx = gatheredCandidatesIdxs[processWithLowestDist] # 'Main' process broadcasts the finding to all processes receivedBroadcast = \ comm.one2all_broadcast(0,\ (processWithLowestDist,addedNodeIdx,lowestOverallDistance,)) # If this process is responsible for the newly found shortest path, # then mark this node as visited. if myRank == receivedBroadcast[0]: indexToUpdate = receivedBroadcast[1] - myRank * chunkSize localRemainView[indexToUpdate] = '' # Each process loop over all connections to this last visited node # that they have the visibility to. for iter in range(len(localResultView)): potentialNewDistance = \ myGraphChunk[iter][receivedBroadcast[1]]+receivedBroadcast[2] # If the distances through this node are less than what we know if potentialNewDistance < localResultView[iter]: # Update the local portion of the result list and # remaining nodes list localResultView[iter] = potentialNewDistance localRemainView[iter] = potentialNewDistance # Reduce the final result to a single list and return it reassembled = comm.all2one_reduce(0, localResultView) if myRank == 0: return reassembled else: return []
print('Number of Vertices in the Graph: ', n) print('Number of Edges in the Graph: ', e) print('Use heuristic methods (only applies to Parallel Kruskal MST)?: ', h) print('Run serial Kruskal MST?: ', s) #run the serial Kruskal Algorithm if chosen if s == 1: serial_graph = make_graph_tuple(n, e) #start the timer st = time.time() MST = kruskal_serial(n, serial_graph) print(MST) #stop timer #print 'Total Time: ',(time.time() - st) else: comm = PSim(p) #create nodes, node 0 is the master #master node if comm.rank == 0: #the master node creates the initial graph the_graph = make_graph_tuple(n, e) #start the timer st = time.time() #break up edges and send to other worker nodes to #sort the edges by weight num_edges = e / p #node 0's edges to sort local_edges_to_sort = the_graph[0:num_edges] for i in range(1, p): comm.send(i, the_graph[num_edges * i:(num_edges * i) + num_edges]) #sort own local piece
import heapq from psim import PSim comm = PSim(5) #each process should get a dictionary item adj = {} #the adjacency dictionary-list adj['a'] = [('b', 10), ('c', 3)] adj['b'] = [('c', 1), ('d', 2)] adj['c'] = [('b', 4), ('d', 8), ('e', 2)] adj['d'] = [('e', 7)] adj['e'] = [('d', 9)] d = {} #tells us the weight at any node Q = [] #used by the heapq S = [] #big s V = ['a', 'b', 'c', 'd', 'e'] #all the vertices for v in V: #assign all nodes with value infinity which is represented as 1000 d[v] = 1000 heapq.heappush(Q, (1000, v)) d['a'] = 0 #start is 0 heapq.heapreplace(Q, (0, 'a')) #start is 0 on the heapq while len(Q) != 0: u = heapq.heappop(Q)[1] #this will just give us the vertex name in the tuple (0, 'a') #after a heappop we need to let all the other processes know #print u S.append(u) #put u in big S for v in adj[u]: #for all the neighbors of the minimum node u #v looks like ('b', 10) #print d[v[0]] #we do v[0] because we just want the vertex name of the tuple
newPosa = [] newPosc = [] newPost = [] newPosg = [] #build initial index seed for i in range(numStrings): newPosa.append(0) newPosc.append(0) newPost.append(0) newPosg.append(0) #Now we parallelize the calculating of the Z values #We create 5 seperate processes, 1 master process to direct and control #and 4 other processes to each calculated the Z values comm = PSim(5) #4 bases while True: #rank 0 is the director of the process and controls the loop and what to send if comm.rank == 0: newRow = GetLetter(newPosa, newPosc, newPost, newPosg) newPosa = GetNextIndexes(newRow, Ba) newPosc = GetNextIndexes(newRow, Bc) newPost = GetNextIndexes(newRow, Bt) newPosg = GetNextIndexes(newRow, Bg) #Exit the loop when there are no other values to calculate Z with if len(newPosa) < numStrings and \ len(newPosc) < numStrings and \