def dividetask(data,task,silent=True): data=mpi.broadcast(mpi.world,data,0) nProcs = mpi.world.size chunkSize=len(data)//nProcs extraBits =len(data)%nProcs res=[] allRes=[] # the root node handles the extra pieces: if mpi.world.rank == 0: for i in range(extraBits): elem=data[i] res.append(task(elem)) if not silent: logger.info('task(%d) done %d'%(mpi.world.rank,i+1)) pos=extraBits+mpi.world.rank*chunkSize; for i in range(chunkSize): elem=data[pos] pos += 1 res.append(task(elem)) if not silent: logger.info('task(%d) done %d'%(mpi.world.rank,i+1)) if mpi.world.rank==0: tmp=mpi.gather(mpi.world,res,0) for res in tmp: allRes.extend(res) else: mpi.gather(mpi.world,res,0) return allRes
def dividetask(data, task, silent=True): data = mpi.broadcast(mpi.world, data, 0) nProcs = mpi.world.size chunkSize = len(data) // nProcs extraBits = len(data) % nProcs res = [] allRes = [] # the root node handles the extra pieces: if mpi.world.rank == 0: for i in range(extraBits): elem = data[i] res.append(task(elem)) if not silent: logger.info('task(%d) done %d' % (mpi.world.rank, i + 1)) pos = extraBits + mpi.world.rank * chunkSize for i in range(chunkSize): elem = data[pos] pos += 1 res.append(task(elem)) if not silent: logger.info('task(%d) done %d' % (mpi.world.rank, i + 1)) if mpi.world.rank == 0: tmp = mpi.gather(mpi.world, res, 0) for res in tmp: allRes.extend(res) else: mpi.gather(mpi.world, res, 0) return allRes
def synchronize_logs(self): """Transfer data from client ranks to the head rank. Must be called on all ranks simultaneously.""" if self.mpi_comm is None: return from boost.mpi import gather if self.mpi_comm.rank == self.head_rank: for rank_data in gather(self.mpi_comm, None, self.head_rank)[1:]: for name, rows in rank_data: self.get_table(name).insert_rows(rows) if self.db_conn is not None: for row in rows: self.db_conn.execute( "insert into %s values (?,?,?)" % name, row) else: # send non-head data away gather(self.mpi_comm, [(name, self.get_table(name).data) for name, qdat in self.quantity_data.iteritems()], self.head_rank) # and erase it for qname in self.quantity_data.iterkeys(): self.get_table(qname).clear()
def _watch_tick(self): def get_last_value(table): if table: return table.data[-1][2] else: return 0 data_block = dict((qname, get_last_value(self.get_table(qname))) for qname in self.quantity_data.iterkeys()) if self.mpi_comm is not None: from boost.mpi import broadcast, gather gathered_data = gather(self.mpi_comm, data_block, self.head_rank) else: gathered_data = [data_block] if self.rank == self.head_rank: values = {} for data_block in gathered_data: for name, value in data_block.iteritems(): values.setdefault(name, []).append(value) if self.watches: print " | ".join( "%s=%g" % (watch.expr, watch.compiled( *[dd.agg_func(values[dd.name]) for dd in watch.dep_data])) for watch in self.watches ) ticks_per_sec = self.tick_count/max(1, time()-self.start_time) self.next_watch_tick = self.tick_count + int(max(1, ticks_per_sec)) if self.mpi_comm is not None: self.next_watch_tick = broadcast(self.mpi_comm, self.next_watch_tick, self.head_rank)
def perceptron_parallel(epoch, indices, blob, weights = None, valid_feature_names = None): """ Implements parallelized version of perceptron training for structured outputs (Collins, 2002; McDonald, 2010). """ # Which processor am I? myRank = mpi.rank # Let processor 0 be the master. masterRank = 0 # How many processors are there? nProcs = mpi.size ########################################## # Keep track of time to train this epoch ########################################## startTime = time.time() # Restart with weights from last epoch or 0. # Will ignore any weights passed during function call. weights_restart_filename = '%s/training-restart.%s' % (tmpdir, str(mpi.rank)) if os.path.isfile(weights_restart_filename): weights_restart_file = open(weights_restart_filename, 'r') weights = cPickle.load(weights_restart_file) weights_restart_file.close() else: # If weights passed during function call is None start with empty. if weights is None or len(weights) == 0: weights = svector.Vector() # Restart with previous running weight sum, also. weights_sum_filename = '%s/training.%s' % (tmpdir, str(mpi.rank)) if os.path.isfile(weights_sum_filename): weights_sum_file = open(weights_sum_filename, 'r') weights_sum = cPickle.load(weights_sum_file) weights_sum_file.close() else: weights_sum = svector.Vector() numChanged = 0 done = False for i, instanceID in enumerate(indices[:FLAGS.subset]): if myRank == i % nProcs: # Assign the current instances we will look at f = blob['f_instances'][instanceID] e = blob['e_instances'][instanceID] etree = blob['etree_instances'][instanceID] gold_str = blob['gold_instances'][instanceID] inverse = None if FLAGS.inverse is not None: inverse = blob['inverse_instances'][instanceID] a1 = None if FLAGS.a1 is not None: a1 = blob['a1_instances'][instanceID] a2 = None if FLAGS.a2 is not None: a2 = blob['a2_instances'][instanceID] ftree = None if FLAGS.ftrees is not None: ftree = blob['ftree_instances'][instanceID] # Preprocess input data # f, e are sequences of words f = f.split() ; e = e.split() # gold is a sequence of f-e link pairs gold = Alignment.Alignment(gold_str) # Initialize model for this instance model = GridAlign.Model(f, e, etree, ftree, instanceID, weights, a1, a2, inverse, LOCAL_FEATURES=blob['localFeatures'], NONLOCAL_FEATURES=blob['nonlocalFeatures'], FLAGS=FLAGS) model.gold = gold # Initialize model with data tables model.pef = blob['pef'] model.pfe = blob['pfe'] # Align the current training instance model.align() ###################################################################### # Weight updating ###################################################################### LEARNING_RATE = FLAGS.learningrate # Set the oracle item oracle = None if FLAGS.oracle == 'gold': oracle = model.oracle elif FLAGS.oracle == 'hope': oracle = model.hope else: sys.stderr.write("ERROR: Unknown oracle class: %s\n" %(FLAGS.oracle)) # Set the hypothesis item hyp = None if FLAGS.hyp == '1best': hyp = model.modelBest elif FLAGS.hyp == 'fear': hyp = model.fear else: sys.stderr.write("ERROR: Unknown hyp class: %s\n" %(FLAGS.hyp)) # Debiasing if FLAGS.debiasing: validate_features(oracle.scoreVector, valid_feature_names) validate_features(hyp.scoreVector, valid_feature_names) deltas = None if set(hyp.links) != set(oracle.links): numChanged += 1 ############################################################### # WEIGHT UPDATES ################################################################ deltas = oracle.scoreVector - hyp.scoreVector weights = weights + LEARNING_RATE*deltas # Even if we didnt update, the current weight vector should count towards the sum! weights_sum += weights # L1 Projection step # if w in [-tau, tau], w -> 0 # else, move w closer to 0 by tau. if FLAGS.tau is not None: for index, w in weights_sum.iteritems(): if w == 0: del weights_sum[index] continue if index[-3:] == '_nb': continue if w > 0 and w <= FLAGS.tau and not FLAGS.negreg: del weights_sum[index] elif w < 0 and w >= (FLAGS.tau * -1): del weights_sum[index] elif w > 0 and w > FLAGS.tau and not FLAGS.negreg: weights_sum[index] -= FLAGS.tau elif w < 0 and w < (FLAGS.tau * -1): weights_sum[index] += FLAGS.tau # Set uniq pickled output file for this process # Holds sum of weights over each iteration for this process output_filename = "%s/training.%s" %(tmpdir, str(mpi.rank)) output_file = open(output_filename,'w') # Dump all weights used during this node's run; to be averaged by master along with others cPickle.dump(weights_sum, output_file, protocol=cPickle.HIGHEST_PROTOCOL) output_file.close() # Remeber just the last weights used for this process; start here next epoch. output_filename_last_weights = "%s/training-restart.%s" %(tmpdir, str(mpi.rank)) output_file_last_weights = open(output_filename_last_weights,'w') cPickle.dump(weights, output_file_last_weights, protocol=cPickle.HIGHEST_PROTOCOL) output_file_last_weights.close() ############################################# # Gather "done" messages from workers ############################################# # Synchronize done = mpi.gather(value=True,root=0) ##################################################################################### # Compute f-measure over all alignments ##################################################################################### masterWeights = svector.Vector() if myRank == masterRank: # Read pickled output for rank in range(nProcs): input_filename = tmpdir+'/training.'+str(rank) input_file = open(input_filename,'r') masterWeights += cPickle.load(input_file) input_file.close() sys.stderr.write("Done reading data.\n") sys.stderr.write("len(masterWeights)= %d\n"%(len(masterWeights))) sys.stderr.flush() ###################################################### # AVERAGED WEIGHTS ###################################################### sys.stderr.write("[%d] Averaging weights.\n" %(mpi.rank)) sys.stderr.flush() masterWeights = masterWeights / (len(indices) * (epoch+1)) # Dump master weights to file # There is only one weight vector in this file at a time. mw = robustWrite(tmpdir+'/weights') cPickle.dump(masterWeights,mw,protocol=cPickle.HIGHEST_PROTOCOL) mw.close() ###################################################################### # All processes read and load new averaged weights ###################################################################### # But make sure worker nodes don't attempt to read from the weights # file before the root node has written it. # Sync-up with a blocking broadcast call ready = mpi.broadcast(value=True, root=0) mw = robustRead(tmpdir+'/weights') masterWeights = cPickle.load(mw) mw.close() ###################################################################### # Print report for this iteration ###################################################################### elapsedTime = time.time() - startTime if myRank == masterRank: # masterRank is printing elapsed time. # May differ at each node. sys.stderr.write("Time: %0.2f\n" %(elapsedTime)) sys.stderr.write("[%d] Finished training.\n" %(mpi.rank)) return masterWeights
def decode_parallel(weights, indices, blob, name="", out=sys.stdout): """ Align some input data in blob with a given weight vector. Report accuracy. """ myRank = mpi.rank masterRank = 0 # How many processors are there? nProcs = mpi.size results = [ ] allResults = None fmeasure = 0.0 ########################################## # Keep track of time to train this epoch ########################################## startTime = time.time() result_file = robustWrite(tmpdir+'/results.'+str(mpi.rank)) for i, instanceID in enumerate(indices[:FLAGS.subset]): if myRank == i % nProcs: # Assign the current instances we will look at f = blob['f_instances'][instanceID] e = blob['e_instances'][instanceID] etree = blob['etree_instances'][instanceID] if FLAGS.train: gold_str = blob['gold_instances'][instanceID] gold = Alignment.Alignment(gold_str) ftree = None if FLAGS.ftrees is not None: ftree = blob['ftree_instances'][instanceID] inverse = None if FLAGS.inverse is not None: inverse = blob['inverse_instances'][instanceID] a1 = None if FLAGS.a1 is not None: a1 = blob['a1_instances'][instanceID] a2 = None if FLAGS.a2 is not None: a2 = blob['a2_instances'][instanceID] # Prepare input data. # f, e are sequences of words f = f.split() e = e.split() # Initialize model for this instance model = GridAlign.Model(f, e, etree, ftree, instanceID, weights, a1, a2, inverse, DECODING=True, LOCAL_FEATURES=blob['localFeatures'], NONLOCAL_FEATURES=blob['nonlocalFeatures'], FLAGS=FLAGS) if FLAGS.train: model.gold = gold # Initialize model with data tables model.pef = blob['pef'] model.pfe = blob['pfe'] # Align the current training instance # FOR PROFILING: cProfile.run('model.align(1)','profile.out') model.align() # Dump intermediate chunk to disk. Reassemble later. if FLAGS.train: cPickle.dump((model.modelBest.links, model.gold.links_dict), result_file, protocol=cPickle.HIGHEST_PROTOCOL) elif FLAGS.align: cPickle.dump(model.modelBest.links, result_file, protocol=cPickle.HIGHEST_PROTOCOL) result_file.close() done = mpi.gather(value=True, root=0) # REDUCE HERE if myRank == masterRank: # Open result files for reading resultFiles = { } for i in range(nProcs): resultFiles[i] = open(tmpdir+'/results.'+str(i),'r') if FLAGS.train: ########################################################################## # Compute f-measure over all alignments ########################################################################## numCorrect = 0 numModelLinks = 0 numGoldLinks = 0 for i, instanceID in enumerate(indices[:FLAGS.subset]): # What node stored instance i node = i % nProcs # Retrieve result from instance i resultTuple = cPickle.load(resultFiles[node]) modelBest = resultTuple[0] gold = resultTuple[1] # Update F-score counts numCorrect_, numModelLinks_, numGoldLinks_ = f1accumulator(modelBest, gold) numCorrect += numCorrect_ numModelLinks += numModelLinks_ numGoldLinks += numGoldLinks_ # Compute F-measure, Precision, and Recall fmeasure, precision, recall = f1score(numCorrect, numModelLinks, numGoldLinks) elapsedTime = time.time() - startTime ###################################################################### # Print report for this iteration ###################################################################### sys.stderr.write("Time: "+str(elapsedTime)+"\n") sys.stderr.write("\n") sys.stderr.write('F-score-%s: %1.5f\n' % (name, fmeasure)) sys.stderr.write('Precision-%s: %1.5f\n' % (name, precision)) sys.stderr.write('Recall-%s: %1.5f\n' % (name, recall)) sys.stderr.write('# Correct: %d\n' % (numCorrect)) sys.stderr.write('# Me Total: %d\n' % (numModelLinks)) sys.stderr.write('# Gold Total: %d\n' % (numGoldLinks)) sys.stderr.write("[%d] Finished decoding.\n" %(myRank)) else: for i, instanceID in enumerate(indices): node = i % nProcs modelBestLinks = cPickle.load(resultFiles[node]) out.write("%s\n" %(" ".join(map(lambda link: "%s-%s" %(link[0], link[1]), modelBestLinks)))) # CLEAN UP for i in range(nProcs): resultFiles[i].close() return
''' Created on Nov 11, 2010 @author: joel ''' import boost.mpi as mpi import numpy as np from time import time N = 1000000 #a = np.arange(N, dtype=float) a = [np.arange(10) for i in range(N/10)] if mpi.world.rank == 0: ts = [] ts.append(time()) mpi.broadcast(mpi.world, a, root=0) ts.append(time()) g = mpi.gather(mpi.world, a, root=0) ts.append(time()) ta = np.array(ts) print "Times taken: ",(ta[1:] - ta[:-1]) else: mpi.broadcast(mpi.world, a, root=0) mpi.gather(mpi.world, a, root=0)