def spout(self):
self.distribute(mpi.world.rank)
self.sendFromSpout(self.MAPINTAG,self.spout_buffer) #send the rest
#tells mappers to finish receiving data
reqs = mpi.RequestList()
for mapper in self.mappers():
done_msg = mpi.world.recv(mapper,self.MAPREQTAG)
reqs.append(mpi.world.isend(mapper,self.MAPINTAG,"EOF"))
mpi.wait_all(reqs)
if self.debug_ == 1: sys.stderr.write("data sent from spout:"+str(mpi.world.rank)+"\n")
def rank0():
sent_histories = (mpi.size - 1) * 15
print "sending %d packets on their way" % sent_histories
send_reqs = mpi.RequestList()
for i in range(sent_histories):
dest = random.randrange(1, mpi.size)
send_reqs.append(mpi.world.isend(dest, TAG_DATA, []))
mpi.wait_all(send_reqs)
completed_histories = []
progress_reports = {}
dead_kids = []
tgl = TagGroupListener(
mpi.world, [TAG_DATA, TAG_DEBUG, TAG_PROGRESS_REPORT, TAG_TERMINATE])
def is_complete():
for i in progress_reports.values():
if i != sent_histories:
return False
return len(dead_kids) == mpi.size - 1
while True:
status, data = tgl.wait()
if status.tag == TAG_DATA:
# print "received completed history %s from %d" % (data, status.source)
completed_histories.append(data)
if len(completed_histories) == sent_histories:
print "all histories received, exiting"
for rank in range(1, mpi.size):
mpi.world.send(rank, TAG_TERMINATE, None)
elif status.tag == TAG_PROGRESS_REPORT:
progress_reports[len(data)] = progress_reports.get(len(data),
0) + 1
elif status.tag == TAG_DEBUG:
print "[DBG %d] %s" % (status.source, data)
elif status.tag == TAG_TERMINATE:
dead_kids.append(status.source)
else:
print "unexpected tag %d from %d" % (status.tag, status.source)
if is_complete():
break
print "OK"
def rank0():
sent_histories = (mpi.size-1)*15
print "sending %d packets on their way" % sent_histories
send_reqs = mpi.RequestList()
for i in range(sent_histories):
dest = random.randrange(1, mpi.size)
send_reqs.append(mpi.world.isend(dest, TAG_DATA, []))
mpi.wait_all(send_reqs)
completed_histories = []
progress_reports = {}
dead_kids = []
tgl = TagGroupListener(mpi.world,
[TAG_DATA, TAG_DEBUG, TAG_PROGRESS_REPORT, TAG_TERMINATE])
def is_complete():
for i in progress_reports.values():
if i != sent_histories:
return False
return len(dead_kids) == mpi.size-1
while True:
status, data = tgl.wait()
if status.tag == TAG_DATA:
#print "received completed history %s from %d" % (data, status.source)
completed_histories.append(data)
if len(completed_histories) == sent_histories:
print "all histories received, exiting"
for rank in range(1, mpi.size):
mpi.world.send(rank, TAG_TERMINATE, None)
elif status.tag == TAG_PROGRESS_REPORT:
progress_reports[len(data)] = progress_reports.get(len(data), 0) + 1
elif status.tag == TAG_DEBUG:
print "[DBG %d] %s" % (status.source, data)
elif status.tag == TAG_TERMINATE:
dead_kids.append(status.source)
else:
print "unexpected tag %d from %d" % (status.tag, status.source)
if is_complete():
break
print "OK"
def master(self):
self.distribute(mpi.world.rank)
self.sendFromSpout(self.MAPINTAG,self.spout_buffer) #send the rest
#tells mappers to finish receiving data
reqs = mpi.RequestList()
for mapper in self.mappers():
done_msg = mpi.world.recv(mapper,self.MAPREQTAG)
reqs.append(mpi.world.isend(mapper,self.MAPINTAG,"EOF"))
mpi.wait_all(reqs)
if self.debug_ == 1: sys.stderr.write("data sent from master\n")
#receives file names from mappers
files = {}
mapnum_rest = len(self.mappers())
while mapnum_rest != 0:
status = mpi.world.iprobe(tag=self.MAPOUTTAG)
while status is None:
time.sleep(0.1)
status = mpi.world.iprobe(tag=self.MAPOUTTAG)
mapper = status.source
mapnum_rest -= 1
if self.debug_ == 1: sys.stderr.write("mapper %d -> master (shuffled file)\n" % mapper)
f = mpi.world.recv(mapper,self.MAPOUTTAG)
for rkey in f.keys():
if files.has_key(rkey):
files[rkey][str(mapper)] = f[rkey]
else:
files[rkey] = {str(mapper):f[rkey]}
#sends file names to reducers
if self.debug_ == 1: sys.stderr.write("master -> reducer (file list)\n")
for rkey in files.keys():
mpi.world.send(int(rkey), self.REDINTAG, files[rkey])
#receives file request from reducers
if self.debug_ == 1: sys.stderr.write("reducer -> master -> mapper(file request)\n")
for reducer in self.reducers():
fnames = mpi.world.recv(reducer,self.MAPFILEREQTAG)
for mkey in fnames.keys():
for fname in fnames[mkey]:
mpi.world.send(int(mkey),self.MAPFILEREQTAG,(reducer,fname))
if self.debug_ == 1: sys.stderr.write("reduce step\n")
#receives file names from reducers and concatinates them
fout = open(self.out_file,"w")
rednum_rest = len(self.reducers())
while rednum_rest != 0:
status = mpi.world.iprobe(tag=self.REDOUTTAG)
while status is None:
time.sleep(0.1)
status = mpi.world.iprobe(tag=self.REDOUTTAG)
reducer = status.source
rednum_rest -= 1
if self.debug_ == 1: sys.stderr.write("reducer %d -> master (reduced data)\n" % reducer)
fname = mpi.world.recv(reducer,self.REDOUTTAG)
if os.path.exists(fname):
for line in open(fname,"r"):
fout.write(line)
try:
os.remove(fname)
except:
self.setError(fname+" might not be deleted")
else:
mpi.world.send(reducer,self.REDFILEREQTAG,fname)
content = mpi.world.recv(reducer,self.REDFILETAG)
while content != "":
fout.write(content)
content = mpi.world.recv(reducer,self.REDFILETAG)
if self.debug_ == 1: sys.stderr.write("shutting down\n")
#shuts down mapper
for mapper in self.mappers():
mpi.world.send(mapper,self.MAPFILEREQTAG,"")
#shuts down reducer
for reducer in self.reducers():
mpi.world.send(reducer,self.REDFILEREQTAG,"")
#gathers error message from mappers and reducers
for child in self.mappers()+self.reducers():
self.error_.extend(mpi.world.recv(child,self.FINISHTAG))
for error in self.error_:
print error