def spout(self): self.distribute(mpi.world.rank) self.sendFromSpout(self.MAPINTAG,self.spout_buffer) #send the rest #tells mappers to finish receiving data reqs = mpi.RequestList() for mapper in self.mappers(): done_msg = mpi.world.recv(mapper,self.MAPREQTAG) reqs.append(mpi.world.isend(mapper,self.MAPINTAG,"EOF")) mpi.wait_all(reqs) if self.debug_ == 1: sys.stderr.write("data sent from spout:"+str(mpi.world.rank)+"\n")
def rank0(): sent_histories = (mpi.size - 1) * 15 print "sending %d packets on their way" % sent_histories send_reqs = mpi.RequestList() for i in range(sent_histories): dest = random.randrange(1, mpi.size) send_reqs.append(mpi.world.isend(dest, TAG_DATA, [])) mpi.wait_all(send_reqs) completed_histories = [] progress_reports = {} dead_kids = [] tgl = TagGroupListener( mpi.world, [TAG_DATA, TAG_DEBUG, TAG_PROGRESS_REPORT, TAG_TERMINATE]) def is_complete(): for i in progress_reports.values(): if i != sent_histories: return False return len(dead_kids) == mpi.size - 1 while True: status, data = tgl.wait() if status.tag == TAG_DATA: # print "received completed history %s from %d" % (data, status.source) completed_histories.append(data) if len(completed_histories) == sent_histories: print "all histories received, exiting" for rank in range(1, mpi.size): mpi.world.send(rank, TAG_TERMINATE, None) elif status.tag == TAG_PROGRESS_REPORT: progress_reports[len(data)] = progress_reports.get(len(data), 0) + 1 elif status.tag == TAG_DEBUG: print "[DBG %d] %s" % (status.source, data) elif status.tag == TAG_TERMINATE: dead_kids.append(status.source) else: print "unexpected tag %d from %d" % (status.tag, status.source) if is_complete(): break print "OK"
def rank0(): sent_histories = (mpi.size-1)*15 print "sending %d packets on their way" % sent_histories send_reqs = mpi.RequestList() for i in range(sent_histories): dest = random.randrange(1, mpi.size) send_reqs.append(mpi.world.isend(dest, TAG_DATA, [])) mpi.wait_all(send_reqs) completed_histories = [] progress_reports = {} dead_kids = [] tgl = TagGroupListener(mpi.world, [TAG_DATA, TAG_DEBUG, TAG_PROGRESS_REPORT, TAG_TERMINATE]) def is_complete(): for i in progress_reports.values(): if i != sent_histories: return False return len(dead_kids) == mpi.size-1 while True: status, data = tgl.wait() if status.tag == TAG_DATA: #print "received completed history %s from %d" % (data, status.source) completed_histories.append(data) if len(completed_histories) == sent_histories: print "all histories received, exiting" for rank in range(1, mpi.size): mpi.world.send(rank, TAG_TERMINATE, None) elif status.tag == TAG_PROGRESS_REPORT: progress_reports[len(data)] = progress_reports.get(len(data), 0) + 1 elif status.tag == TAG_DEBUG: print "[DBG %d] %s" % (status.source, data) elif status.tag == TAG_TERMINATE: dead_kids.append(status.source) else: print "unexpected tag %d from %d" % (status.tag, status.source) if is_complete(): break print "OK"
def master(self): self.distribute(mpi.world.rank) self.sendFromSpout(self.MAPINTAG,self.spout_buffer) #send the rest #tells mappers to finish receiving data reqs = mpi.RequestList() for mapper in self.mappers(): done_msg = mpi.world.recv(mapper,self.MAPREQTAG) reqs.append(mpi.world.isend(mapper,self.MAPINTAG,"EOF")) mpi.wait_all(reqs) if self.debug_ == 1: sys.stderr.write("data sent from master\n") #receives file names from mappers files = {} mapnum_rest = len(self.mappers()) while mapnum_rest != 0: status = mpi.world.iprobe(tag=self.MAPOUTTAG) while status is None: time.sleep(0.1) status = mpi.world.iprobe(tag=self.MAPOUTTAG) mapper = status.source mapnum_rest -= 1 if self.debug_ == 1: sys.stderr.write("mapper %d -> master (shuffled file)\n" % mapper) f = mpi.world.recv(mapper,self.MAPOUTTAG) for rkey in f.keys(): if files.has_key(rkey): files[rkey][str(mapper)] = f[rkey] else: files[rkey] = {str(mapper):f[rkey]} #sends file names to reducers if self.debug_ == 1: sys.stderr.write("master -> reducer (file list)\n") for rkey in files.keys(): mpi.world.send(int(rkey), self.REDINTAG, files[rkey]) #receives file request from reducers if self.debug_ == 1: sys.stderr.write("reducer -> master -> mapper(file request)\n") for reducer in self.reducers(): fnames = mpi.world.recv(reducer,self.MAPFILEREQTAG) for mkey in fnames.keys(): for fname in fnames[mkey]: mpi.world.send(int(mkey),self.MAPFILEREQTAG,(reducer,fname)) if self.debug_ == 1: sys.stderr.write("reduce step\n") #receives file names from reducers and concatinates them fout = open(self.out_file,"w") rednum_rest = len(self.reducers()) while rednum_rest != 0: status = mpi.world.iprobe(tag=self.REDOUTTAG) while status is None: time.sleep(0.1) status = mpi.world.iprobe(tag=self.REDOUTTAG) reducer = status.source rednum_rest -= 1 if self.debug_ == 1: sys.stderr.write("reducer %d -> master (reduced data)\n" % reducer) fname = mpi.world.recv(reducer,self.REDOUTTAG) if os.path.exists(fname): for line in open(fname,"r"): fout.write(line) try: os.remove(fname) except: self.setError(fname+" might not be deleted") else: mpi.world.send(reducer,self.REDFILEREQTAG,fname) content = mpi.world.recv(reducer,self.REDFILETAG) while content != "": fout.write(content) content = mpi.world.recv(reducer,self.REDFILETAG) if self.debug_ == 1: sys.stderr.write("shutting down\n") #shuts down mapper for mapper in self.mappers(): mpi.world.send(mapper,self.MAPFILEREQTAG,"") #shuts down reducer for reducer in self.reducers(): mpi.world.send(reducer,self.REDFILEREQTAG,"") #gathers error message from mappers and reducers for child in self.mappers()+self.reducers(): self.error_.extend(mpi.world.recv(child,self.FINISHTAG)) for error in self.error_: print error