def Dijkstra_p(G,start,end=None):
D = {} # dictionary of final distances
P = {} # dictionary of predecessors
Q = priorityDictionary() # est.dist. of non-final vert.
Q[start] = 0
comm = PSim(len(Q)+1)
if comm.rank==0:
i = 1
for v in Q:
D[v] = Q[v]
if v == end: break
comm.send(i, v)
i = i + 1
else:
v = comm.recv(0)
for w in G[v]:
vwLength = D[v] + G[v][w]
if w in D:
if vwLength < D[w]:
raise ValueError, "Dijkstra: found better path to already-final vertex"
elif w not in Q or vwLength < Q[w]:
Q[w] = vwLength
P[w] = v
return (D,P)
def test():
comm=PSim(5,SWITCH)
if comm.rank==0: print 'start test'
a=sum(comm.all2all_broadcast(comm.rank))
comm.barrier()
b=comm.all2all_reduce(comm.rank)
if a!=10 or a!=b:
print 'from process', comm.rank
raise Exception
if comm.rank==0: print 'test passed'
def scalar_product_test2(n, p):
comm = PSim(p)
a = b = None
if comm.rank == 0:
a = [random.random() for i in range(n)]
b = [random.random() for i in range(n)]
a = comm.one2all_scatter(0, a)
b = comm.one2all_scatter(0, b)
scalar = sum(a[i] * b[i] for i in range(len(a)))
scalar = comm.all2one_reduce(0, scalar)
if comm.rank == 0:
print(scalar)
def scalar_product_test2(n,p):
import random
from psim import PSim
comm = PSim(p)
a = b = None
if comm.rank==0:
a = [random.random() for i in range(n)]
b = [random.random() for i in range(n)]
a = comm.one2all_scatter(0,a)
b = comm.one2all_scatter(0,b)
scalar = sum(a[i]*b[i] for i in range(len(a)))
scalar = comm.all2one_reduce(0,scalar)
if comm.rank == 0:
print scalar
def Parallel(V, E, w, s): l = V.keys() """ Create a new process for each start vector. Note that there is an "extra" because the root process does NOT handle any work - it just scatters and gathers data. """ comm = PSim(len(l) + 1) # Scatter if comm.rank==0: # Loop through all of the processes for i in range(1, len(l)+1): comm.send(i, l[i-1]) else: curr = comm.recv(0) res = Dijkstra(V, curr) comm.send(0, res) # Gather if comm.rank==0: # Loop through all of the processes for i in range(1, len(l)+1): res = comm.recv(i) print "" print "Start: " + str(l[i-1]) print "Path: " + str(res)
def mergesort_test(n, p):
comm = PSim(p)
if comm.rank == 0:
data = [random.random() for i in range(n)]
comm.send(1, data[n/2:])
mergesort(data, 0, n/2)
data[n/2:] = comm.recv(1)
merge(data, 0, n/2, n)
print(data)
else:
data = comm.recv(0)
mergesort(data)
comm.send(0, data)
def test():
comm = PSim(5, SWITCH)
if comm.rank == 0: print 'start test'
a = sum(comm.all2all_broadcast(comm.rank))
comm.barrier()
b = comm.all2all_reduce(comm.rank)
if a != 10 or a != b:
print 'from process', comm.rank
raise Exception
if comm.rank == 0: print 'test passed'
def mergesort_test(n,p):
import random
from psim import PSim
comm = PSim(p)
if comm.rank==0:
data = [random.random() for i in range(n)]
comm.send(1, data[n/2:])
mergesort(data,0,n/2)
data[n/2:] = comm.recv(1)
merge(data,0,n/2,n)
print data
else:
data = comm.recv(0)
mergesort(data)
comm.send(0,data)
def scalar_product_test1(n, p):
comm = PSim(p)
h = n / p
if comm.rank == 0:
a = [random.random() for i in range(n)]
b = [random.random() for i in range(n)]
for k in range(1, p):
comm.send(k, a[k * h:k * h + h])
comm.send(k, b[k * h:k * h + h])
else:
a = comm.recv(0)
b = comm.recv(0)
scalar = sum(a[i] * b[i] for i in range(h))
if comm.rank == 0:
for k in range(1, p):
scalar += comm.recv(k)
print(scalar)
else:
comm.send(0, scalar)
def Dijkstra_pf(graph, start):
vertices, links = graph
P = [PrimVertex(i, links) for i in vertices]
Q = [P[i] for i in vertices if not i == start]
vertex = P[start]
vertex.closest_dist = 0
comm = PSim(len(vertices) + 1)
while Q:
i = 1
for neighbor_id, length in vertex.neighbors:
if comm.rank == 0:
# comm.send(i, (P[neighbor_id], length))
comm.send(i, (neighbor_id, length))
else:
# neighbor, length = comm.recv(0)
nid, lth = comm.recv(0)
neighbor = P[nid]
dist = lth + vertex.closest_dist
if neighbor in Q and dist < neighbor.closest_dist:
neighbor.closest = vertex
neighbor.closest_dist = dist
comm.send(0, (nid, neighbor))
i = i + 1
if comm.rank == 0:
l = []
# Gather the message from each iteration
for i in range(1, i):
nid, r = comm.recv(i)
l.append(r)
neighbor = P[nid]
if r.closest_dist < neighbor.closest_dist:
neighbor.closest = vertex
neighbor.closest_dist = r.closest_dist
heapify(Q)
vertex = heappop(Q)
# for v in P: print v.closest
return [(v.id, v.closest.id, v.closest_dist) for v in P if not v.id == start]
"""if comm.rank==0:
def scalar_product_test1(n,p):
import random
from psim import PSim
comm = PSim(p)
h = n/p
if comm.rank==0:
a = [random.random() for i in range(n)]
b = [random.random() for i in range(n)]
for k in range(1,p):
comm.send(k, a[k*h:k*h+h])
comm.send(k, b[k*h:k*h+h])
else:
a = comm.recv(0)
b = comm.recv(0)
scalar = sum(a[i]*b[i] for i in range(h))
if comm.rank == 0:
for k in range(1,p):
scalar += comm.recv(k)
print scalar
else:
comm.send(0,scalar)
def bb(adjacency, p=1):
n = len(adjacency)
comm = PSim(p)
Q = []
path = [0]
Q.append(Vertex(path))
bound = float("inf')
optimal = None
local_vertices = comm.one2all_scatter(0, range(n))
while True:
if comm.rank == 0:
vertex = Q.pop() if Q else None
else:
vertex = None
vertex = comm.one2all_broadcast(0, vertex)
if vertex is None:
break
P = []
for k in local_vertices:
if not k in vertex.path:
new_path = vertex.path+[k]
new_path_length = weight(new_path, adjacency)
if new_path_length<bound:
if len(new_path) == n:
new_path.append(new_path[0])
new_path_length = weight(new_path, adjacency)
if new_path_length<bound:
bound = new_path_length # bcast
optimal = new_path # bcast
else:
new_vertex = Vertex(new_path)
P.append(new_vertex) # fix this
print(new_path, new_path_length)
x = (bound, optimal)
x = comm.all2all_reduce(x, lambda a, b: min(a, b))
(bound, optimal) = x
P = comm.all2one_collect(0, P)
if comm.rank == 0:
for item in P:
Q+=item
return optimal, bound
# Begin program execution by supplying n verticies, indexSize (5 would
# mean vertex names would consist of 5 random characters, such as ABCDE;
# the higher the number, the more unique vertex names that are available),
# maxEdges indicates the maximum number of edges from any vertex, and min/max
# cost sets the range of costs for the randomly generated cost data.
n = 4096
indexSize = 4
maxEdges = 4 #must be less than n
minCost = 1
maxCost = 100
# Set the dimensions and processors for this program execution.
d = 4
p = 2**d
comm = PSim(p)
#Generate the random values including source/destination verticies
if comm.rank==0:
start = time.time()
index,revIndex,graph,distances = genData(n,indexSize,maxEdges,minCost,maxCost)
source = revIndex[random.randint(0,len(revIndex)-1)]
destination = source
while(destination == source):
destination = revIndex[random.randint(0,len(revIndex)-1)]
else:
index,revIndex,graph,distances = None,None,None,None
source = None
destination = None
#Distribute a portion of the data to all nodes
print 'Number of Vertices in the Graph: ', n print 'Number of Edges in the Graph: ', e print 'Use heuristic methods (only applies to Parallel Kruskal MST)?: ', h print 'Run serial Kruskal MST?: ', s #run the serial Kruskal Algorithm if chosen if s == 1: serial_graph = make_graph_tuple(n,e) #start the timer st = time.time() MST = kruskal_serial(n, serial_graph) print MST #stop timer #print 'Total Time: ',(time.time() - st) else: comm = PSim(p) #create nodes, node 0 is the master #master node if comm.rank == 0: #the master node creates the initial graph the_graph = make_graph_tuple(n,e) #start the timer st = time.time() #break up edges and send to other worker nodes to #sort the edges by weight num_edges = e / p #node 0's edges to sort local_edges_to_sort = the_graph[0:num_edges] for i in range (1,p): comm.send(i, the_graph[num_edges*i:(num_edges*i)+num_edges]) #sort own local piece
# Begin program execution by supplying n verticies, indexSize (5 would
# mean vertex names would consist of 5 random characters, such as ABCDE;
# the higher the number, the more unique vertex names that are available),
# maxEdges indicates the maximum number of edges from any vertex, and min/max
# cost sets the range of costs for the randomly generated cost data.
n = 4096
indexSize = 4
maxEdges = 4 #must be less than n
minCost = 1
maxCost = 100
# Set the dimensions and processors for this program execution.
d = 4
p = 2**d
comm = PSim(p)
#Generate the random values including source/destination verticies
if comm.rank == 0:
start = time.time()
index, revIndex, graph, distances = genData(n, indexSize, maxEdges,
minCost, maxCost)
source = revIndex[random.randint(0, len(revIndex) - 1)]
destination = source
while (destination == source):
destination = revIndex[random.randint(0, len(revIndex) - 1)]
else:
index, revIndex, graph, distances = None, None, None, None
source = None
destination = None
def dijkstra_p(g, source, p):
"""
PYsim parallel implementation of dijkstra's algorithm
Result: One source shourtest paths to all nodes
Implementation parallelizes finding minimums and
updating the results (inner loop is parallelized)
Uses the method of collecting of intermediate minimums and
broadcasting of newly found path to all processes
"""
# Initialize communicator
comm = PSim(p)
myRank = comm.rank
chunkSize = len(g) / p
# Scatter the graph by partitioning it one dimensionally,
# along with known distances to source, and remaining node information
myGraphChunk = comm.one2all_scatter(0, g)
localResultView = comm.one2all_scatter(0, g[source])
localRemainView = comm.one2all_scatter(0, g[source])
# Each process will iterate q times, where q is number of nodes in graph
for q in range(len(g)):
# Get shortest path to locally un-visited nodes
lowestLocalDist = min(localRemainView)
lowestLocalDistIdx = localRemainView.index(lowestLocalDist)
# Collect the lowest distances and node indexes from all processes
gatheredCandidatesDist = comm.all2one_collect(0, lowestLocalDist)
gatheredCandidatesIdxs = comm.all2one_collect(0, lowestLocalDistIdx + myRank * chunkSize)
processWithLowestDist = -1
addedNodeIdx = -1
lowestOverallDistance = -1
# One 'main' process finds the global minimum
if myRank == 0:
lowestOverallDistance = min(gatheredCandidatesDist)
processWithLowestDist = gatheredCandidatesDist.index(lowestOverallDistance)
addedNodeIdx = gatheredCandidatesIdxs[processWithLowestDist]
# 'Main' process broadcasts the finding to all processes
receivedBroadcast = comm.one2all_broadcast(0, (processWithLowestDist, addedNodeIdx, lowestOverallDistance))
# If this process is responsible for the newly found shortest path,
# then mark this node as visited.
if myRank == receivedBroadcast[0]:
indexToUpdate = receivedBroadcast[1] - myRank * chunkSize
localRemainView[indexToUpdate] = ""
# Each process loop over all connections to this last visited node
# that they have the visibility to.
for iter in range(len(localResultView)):
potentialNewDistance = myGraphChunk[iter][receivedBroadcast[1]] + receivedBroadcast[2]
# If the distances through this node are less than what we know
if potentialNewDistance < localResultView[iter]:
# Update the local portion of the result list and
# remaining nodes list
localResultView[iter] = potentialNewDistance
localRemainView[iter] = potentialNewDistance
# Reduce the final result to a single list and return it
reassembled = comm.all2one_reduce(0, localResultView)
if myRank == 0:
return reassembled
else:
return []
def dijkstra_p(g, source, p):
"""
PYsim parallel implementation of dijkstra's algorithm
Result: One source shourtest paths to all nodes
Implementation parallelizes finding minimums and
updating the results (inner loop is parallelized)
Uses the method of collecting of intermediate minimums and
broadcasting of newly found path to all processes
"""
# Initialize communicator
comm = PSim(p)
myRank = comm.rank
chunkSize = len(g) / p
# Scatter the graph by partitioning it one dimensionally,
# along with known distances to source, and remaining node information
myGraphChunk = comm.one2all_scatter(0, g)
localResultView = comm.one2all_scatter(0, g[source])
localRemainView = comm.one2all_scatter(0, g[source])
# Each process will iterate q times, where q is number of nodes in graph
for q in range(len(g)):
# Get shortest path to locally un-visited nodes
lowestLocalDist = min(localRemainView)
lowestLocalDistIdx = localRemainView.index(lowestLocalDist)
# Collect the lowest distances and node indexes from all processes
gatheredCandidatesDist = comm.all2one_collect(0, lowestLocalDist)
gatheredCandidatesIdxs = comm.all2one_collect(
0, lowestLocalDistIdx + myRank * chunkSize)
processWithLowestDist = -1
addedNodeIdx = -1
lowestOverallDistance = -1
# One 'main' process finds the global minimum
if myRank == 0:
lowestOverallDistance = min(gatheredCandidatesDist)
processWithLowestDist = gatheredCandidatesDist.index(
lowestOverallDistance)
addedNodeIdx = gatheredCandidatesIdxs[processWithLowestDist]
# 'Main' process broadcasts the finding to all processes
receivedBroadcast = \
comm.one2all_broadcast(0,\
(processWithLowestDist,addedNodeIdx,lowestOverallDistance,))
# If this process is responsible for the newly found shortest path,
# then mark this node as visited.
if myRank == receivedBroadcast[0]:
indexToUpdate = receivedBroadcast[1] - myRank * chunkSize
localRemainView[indexToUpdate] = ''
# Each process loop over all connections to this last visited node
# that they have the visibility to.
for iter in range(len(localResultView)):
potentialNewDistance = \
myGraphChunk[iter][receivedBroadcast[1]]+receivedBroadcast[2]
# If the distances through this node are less than what we know
if potentialNewDistance < localResultView[iter]:
# Update the local portion of the result list and
# remaining nodes list
localResultView[iter] = potentialNewDistance
localRemainView[iter] = potentialNewDistance
# Reduce the final result to a single list and return it
reassembled = comm.all2one_reduce(0, localResultView)
if myRank == 0:
return reassembled
else:
return []
print('Number of Vertices in the Graph: ', n)
print('Number of Edges in the Graph: ', e)
print('Use heuristic methods (only applies to Parallel Kruskal MST)?: ', h)
print('Run serial Kruskal MST?: ', s)
#run the serial Kruskal Algorithm if chosen
if s == 1:
serial_graph = make_graph_tuple(n, e)
#start the timer
st = time.time()
MST = kruskal_serial(n, serial_graph)
print(MST)
#stop timer
#print 'Total Time: ',(time.time() - st)
else:
comm = PSim(p) #create nodes, node 0 is the master
#master node
if comm.rank == 0:
#the master node creates the initial graph
the_graph = make_graph_tuple(n, e)
#start the timer
st = time.time()
#break up edges and send to other worker nodes to
#sort the edges by weight
num_edges = e / p
#node 0's edges to sort
local_edges_to_sort = the_graph[0:num_edges]
for i in range(1, p):
comm.send(i, the_graph[num_edges * i:(num_edges * i) + num_edges])
#sort own local piece
import heapq
from psim import PSim
comm = PSim(5)
#each process should get a dictionary item
adj = {} #the adjacency dictionary-list
adj['a'] = [('b', 10), ('c', 3)]
adj['b'] = [('c', 1), ('d', 2)]
adj['c'] = [('b', 4), ('d', 8), ('e', 2)]
adj['d'] = [('e', 7)]
adj['e'] = [('d', 9)]
d = {} #tells us the weight at any node
Q = [] #used by the heapq
S = [] #big s
V = ['a', 'b', 'c', 'd', 'e'] #all the vertices
for v in V: #assign all nodes with value infinity which is represented as 1000
d[v] = 1000
heapq.heappush(Q, (1000, v))
d['a'] = 0 #start is 0
heapq.heapreplace(Q, (0, 'a')) #start is 0 on the heapq
while len(Q) != 0:
u = heapq.heappop(Q)[1] #this will just give us the vertex name in the tuple (0, 'a')
#after a heappop we need to let all the other processes know
#print u
S.append(u) #put u in big S
for v in adj[u]: #for all the neighbors of the minimum node u #v looks like ('b', 10)
#print d[v[0]] #we do v[0] because we just want the vertex name of the tuple
newPosa = []
newPosc = []
newPost = []
newPosg = []
#build initial index seed
for i in range(numStrings):
newPosa.append(0)
newPosc.append(0)
newPost.append(0)
newPosg.append(0)
#Now we parallelize the calculating of the Z values
#We create 5 seperate processes, 1 master process to direct and control
#and 4 other processes to each calculated the Z values
comm = PSim(5) #4 bases
while True:
#rank 0 is the director of the process and controls the loop and what to send
if comm.rank == 0:
newRow = GetLetter(newPosa, newPosc, newPost, newPosg)
newPosa = GetNextIndexes(newRow, Ba)
newPosc = GetNextIndexes(newRow, Bc)
newPost = GetNextIndexes(newRow, Bt)
newPosg = GetNextIndexes(newRow, Bg)
#Exit the loop when there are no other values to calculate Z with
if len(newPosa) < numStrings and \
len(newPosc) < numStrings and \
newPosa = []
newPosc = []
newPost = []
newPosg = []
#build initial index seed
for i in range(numStrings):
newPosa.append(0)
newPosc.append(0)
newPost.append(0)
newPosg.append(0)
#Now we parallelize the calculating of the Z values
#We create 5 seperate processes, 1 master process to direct and control
#and 4 other processes to each calculated the Z values
comm = PSim(5) #4 bases
while True:
#rank 0 is the director of the process and controls the loop and what to send
if comm.rank == 0:
newRow = GetLetter(newPosa, newPosc, newPost, newPosg)
newPosa = GetNextIndexes(newRow, Ba)
newPosc = GetNextIndexes(newRow, Bc)
newPost = GetNextIndexes(newRow, Bt)
newPosg = GetNextIndexes(newRow, Bg)
#Exit the loop when there are no other values to calculate Z with
if len(newPosa) < numStrings and \
len(newPosc) < numStrings and \
