def dividetask(data,task,silent=True):
data=mpi.broadcast(mpi.world,data,0)
nProcs = mpi.world.size
chunkSize=len(data)//nProcs
extraBits =len(data)%nProcs
res=[]
allRes=[]
# the root node handles the extra pieces:
if mpi.world.rank == 0:
for i in range(extraBits):
elem=data[i]
res.append(task(elem))
if not silent:
logger.info('task(%d) done %d'%(mpi.world.rank,i+1))
pos=extraBits+mpi.world.rank*chunkSize;
for i in range(chunkSize):
elem=data[pos]
pos += 1
res.append(task(elem))
if not silent:
logger.info('task(%d) done %d'%(mpi.world.rank,i+1))
if mpi.world.rank==0:
tmp=mpi.gather(mpi.world,res,0)
for res in tmp: allRes.extend(res)
else:
mpi.gather(mpi.world,res,0)
return allRes
def dividetask(data, task, silent=True):
data = mpi.broadcast(mpi.world, data, 0)
nProcs = mpi.world.size
chunkSize = len(data) // nProcs
extraBits = len(data) % nProcs
res = []
allRes = []
# the root node handles the extra pieces:
if mpi.world.rank == 0:
for i in range(extraBits):
elem = data[i]
res.append(task(elem))
if not silent:
logger.info('task(%d) done %d' % (mpi.world.rank, i + 1))
pos = extraBits + mpi.world.rank * chunkSize
for i in range(chunkSize):
elem = data[pos]
pos += 1
res.append(task(elem))
if not silent:
logger.info('task(%d) done %d' % (mpi.world.rank, i + 1))
if mpi.world.rank == 0:
tmp = mpi.gather(mpi.world, res, 0)
for res in tmp:
allRes.extend(res)
else:
mpi.gather(mpi.world, res, 0)
return allRes
def synchronize_logs(self):
"""Transfer data from client ranks to the head rank.
Must be called on all ranks simultaneously."""
if self.mpi_comm is None:
return
from boost.mpi import gather
if self.mpi_comm.rank == self.head_rank:
for rank_data in gather(self.mpi_comm, None, self.head_rank)[1:]:
for name, rows in rank_data:
self.get_table(name).insert_rows(rows)
if self.db_conn is not None:
for row in rows:
self.db_conn.execute(
"insert into %s values (?,?,?)" % name, row)
else:
# send non-head data away
gather(self.mpi_comm,
[(name, self.get_table(name).data)
for name, qdat in self.quantity_data.iteritems()],
self.head_rank)
# and erase it
for qname in self.quantity_data.iterkeys():
self.get_table(qname).clear()
def _watch_tick(self):
def get_last_value(table):
if table:
return table.data[-1][2]
else:
return 0
data_block = dict((qname, get_last_value(self.get_table(qname)))
for qname in self.quantity_data.iterkeys())
if self.mpi_comm is not None:
from boost.mpi import broadcast, gather
gathered_data = gather(self.mpi_comm, data_block, self.head_rank)
else:
gathered_data = [data_block]
if self.rank == self.head_rank:
values = {}
for data_block in gathered_data:
for name, value in data_block.iteritems():
values.setdefault(name, []).append(value)
if self.watches:
print " | ".join(
"%s=%g" % (watch.expr, watch.compiled(
*[dd.agg_func(values[dd.name]) for dd in watch.dep_data]))
for watch in self.watches
)
ticks_per_sec = self.tick_count/max(1, time()-self.start_time)
self.next_watch_tick = self.tick_count + int(max(1, ticks_per_sec))
if self.mpi_comm is not None:
self.next_watch_tick = broadcast(self.mpi_comm,
self.next_watch_tick, self.head_rank)
def perceptron_parallel(epoch, indices, blob, weights = None, valid_feature_names = None):
"""
Implements parallelized version of perceptron training for structured outputs
(Collins, 2002; McDonald, 2010).
"""
# Which processor am I?
myRank = mpi.rank
# Let processor 0 be the master.
masterRank = 0
# How many processors are there?
nProcs = mpi.size
##########################################
# Keep track of time to train this epoch
##########################################
startTime = time.time()
# Restart with weights from last epoch or 0.
# Will ignore any weights passed during function call.
weights_restart_filename = '%s/training-restart.%s' % (tmpdir, str(mpi.rank))
if os.path.isfile(weights_restart_filename):
weights_restart_file = open(weights_restart_filename, 'r')
weights = cPickle.load(weights_restart_file)
weights_restart_file.close()
else:
# If weights passed during function call is None start with empty.
if weights is None or len(weights) == 0:
weights = svector.Vector()
# Restart with previous running weight sum, also.
weights_sum_filename = '%s/training.%s' % (tmpdir, str(mpi.rank))
if os.path.isfile(weights_sum_filename):
weights_sum_file = open(weights_sum_filename, 'r')
weights_sum = cPickle.load(weights_sum_file)
weights_sum_file.close()
else:
weights_sum = svector.Vector()
numChanged = 0
done = False
for i, instanceID in enumerate(indices[:FLAGS.subset]):
if myRank == i % nProcs:
# Assign the current instances we will look at
f = blob['f_instances'][instanceID]
e = blob['e_instances'][instanceID]
etree = blob['etree_instances'][instanceID]
gold_str = blob['gold_instances'][instanceID]
inverse = None
if FLAGS.inverse is not None:
inverse = blob['inverse_instances'][instanceID]
a1 = None
if FLAGS.a1 is not None:
a1 = blob['a1_instances'][instanceID]
a2 = None
if FLAGS.a2 is not None:
a2 = blob['a2_instances'][instanceID]
ftree = None
if FLAGS.ftrees is not None:
ftree = blob['ftree_instances'][instanceID]
# Preprocess input data
# f, e are sequences of words
f = f.split() ; e = e.split()
# gold is a sequence of f-e link pairs
gold = Alignment.Alignment(gold_str)
# Initialize model for this instance
model = GridAlign.Model(f, e, etree, ftree, instanceID, weights, a1, a2,
inverse, LOCAL_FEATURES=blob['localFeatures'],
NONLOCAL_FEATURES=blob['nonlocalFeatures'],
FLAGS=FLAGS)
model.gold = gold
# Initialize model with data tables
model.pef = blob['pef']
model.pfe = blob['pfe']
# Align the current training instance
model.align()
######################################################################
# Weight updating
######################################################################
LEARNING_RATE = FLAGS.learningrate
# Set the oracle item
oracle = None
if FLAGS.oracle == 'gold':
oracle = model.oracle
elif FLAGS.oracle == 'hope':
oracle = model.hope
else:
sys.stderr.write("ERROR: Unknown oracle class: %s\n" %(FLAGS.oracle))
# Set the hypothesis item
hyp = None
if FLAGS.hyp == '1best':
hyp = model.modelBest
elif FLAGS.hyp == 'fear':
hyp = model.fear
else:
sys.stderr.write("ERROR: Unknown hyp class: %s\n" %(FLAGS.hyp))
# Debiasing
if FLAGS.debiasing:
validate_features(oracle.scoreVector, valid_feature_names)
validate_features(hyp.scoreVector, valid_feature_names)
deltas = None
if set(hyp.links) != set(oracle.links):
numChanged += 1
###############################################################
# WEIGHT UPDATES
################################################################
deltas = oracle.scoreVector - hyp.scoreVector
weights = weights + LEARNING_RATE*deltas
# Even if we didnt update, the current weight vector should count towards the sum!
weights_sum += weights
# L1 Projection step
# if w in [-tau, tau], w -> 0
# else, move w closer to 0 by tau.
if FLAGS.tau is not None:
for index, w in weights_sum.iteritems():
if w == 0:
del weights_sum[index]
continue
if index[-3:] == '_nb':
continue
if w > 0 and w <= FLAGS.tau and not FLAGS.negreg:
del weights_sum[index]
elif w < 0 and w >= (FLAGS.tau * -1):
del weights_sum[index]
elif w > 0 and w > FLAGS.tau and not FLAGS.negreg:
weights_sum[index] -= FLAGS.tau
elif w < 0 and w < (FLAGS.tau * -1):
weights_sum[index] += FLAGS.tau
# Set uniq pickled output file for this process
# Holds sum of weights over each iteration for this process
output_filename = "%s/training.%s" %(tmpdir, str(mpi.rank))
output_file = open(output_filename,'w')
# Dump all weights used during this node's run; to be averaged by master along with others
cPickle.dump(weights_sum, output_file, protocol=cPickle.HIGHEST_PROTOCOL)
output_file.close()
# Remeber just the last weights used for this process; start here next epoch.
output_filename_last_weights = "%s/training-restart.%s" %(tmpdir, str(mpi.rank))
output_file_last_weights = open(output_filename_last_weights,'w')
cPickle.dump(weights, output_file_last_weights, protocol=cPickle.HIGHEST_PROTOCOL)
output_file_last_weights.close()
#############################################
# Gather "done" messages from workers
#############################################
# Synchronize
done = mpi.gather(value=True,root=0)
#####################################################################################
# Compute f-measure over all alignments
#####################################################################################
masterWeights = svector.Vector()
if myRank == masterRank:
# Read pickled output
for rank in range(nProcs):
input_filename = tmpdir+'/training.'+str(rank)
input_file = open(input_filename,'r')
masterWeights += cPickle.load(input_file)
input_file.close()
sys.stderr.write("Done reading data.\n")
sys.stderr.write("len(masterWeights)= %d\n"%(len(masterWeights)))
sys.stderr.flush()
######################################################
# AVERAGED WEIGHTS
######################################################
sys.stderr.write("[%d] Averaging weights.\n" %(mpi.rank))
sys.stderr.flush()
masterWeights = masterWeights / (len(indices) * (epoch+1))
# Dump master weights to file
# There is only one weight vector in this file at a time.
mw = robustWrite(tmpdir+'/weights')
cPickle.dump(masterWeights,mw,protocol=cPickle.HIGHEST_PROTOCOL)
mw.close()
######################################################################
# All processes read and load new averaged weights
######################################################################
# But make sure worker nodes don't attempt to read from the weights
# file before the root node has written it.
# Sync-up with a blocking broadcast call
ready = mpi.broadcast(value=True, root=0)
mw = robustRead(tmpdir+'/weights')
masterWeights = cPickle.load(mw)
mw.close()
######################################################################
# Print report for this iteration
######################################################################
elapsedTime = time.time() - startTime
if myRank == masterRank:
# masterRank is printing elapsed time.
# May differ at each node.
sys.stderr.write("Time: %0.2f\n" %(elapsedTime))
sys.stderr.write("[%d] Finished training.\n" %(mpi.rank))
return masterWeights
def decode_parallel(weights, indices, blob, name="", out=sys.stdout):
"""
Align some input data in blob with a given weight vector. Report accuracy.
"""
myRank = mpi.rank
masterRank = 0
# How many processors are there?
nProcs = mpi.size
results = [ ]
allResults = None
fmeasure = 0.0
##########################################
# Keep track of time to train this epoch
##########################################
startTime = time.time()
result_file = robustWrite(tmpdir+'/results.'+str(mpi.rank))
for i, instanceID in enumerate(indices[:FLAGS.subset]):
if myRank == i % nProcs:
# Assign the current instances we will look at
f = blob['f_instances'][instanceID]
e = blob['e_instances'][instanceID]
etree = blob['etree_instances'][instanceID]
if FLAGS.train:
gold_str = blob['gold_instances'][instanceID]
gold = Alignment.Alignment(gold_str)
ftree = None
if FLAGS.ftrees is not None:
ftree = blob['ftree_instances'][instanceID]
inverse = None
if FLAGS.inverse is not None:
inverse = blob['inverse_instances'][instanceID]
a1 = None
if FLAGS.a1 is not None:
a1 = blob['a1_instances'][instanceID]
a2 = None
if FLAGS.a2 is not None:
a2 = blob['a2_instances'][instanceID]
# Prepare input data.
# f, e are sequences of words
f = f.split()
e = e.split()
# Initialize model for this instance
model = GridAlign.Model(f, e, etree, ftree, instanceID, weights, a1, a2,
inverse, DECODING=True,
LOCAL_FEATURES=blob['localFeatures'],
NONLOCAL_FEATURES=blob['nonlocalFeatures'],
FLAGS=FLAGS)
if FLAGS.train:
model.gold = gold
# Initialize model with data tables
model.pef = blob['pef']
model.pfe = blob['pfe']
# Align the current training instance
# FOR PROFILING: cProfile.run('model.align(1)','profile.out')
model.align()
# Dump intermediate chunk to disk. Reassemble later.
if FLAGS.train:
cPickle.dump((model.modelBest.links, model.gold.links_dict), result_file, protocol=cPickle.HIGHEST_PROTOCOL)
elif FLAGS.align:
cPickle.dump(model.modelBest.links, result_file, protocol=cPickle.HIGHEST_PROTOCOL)
result_file.close()
done = mpi.gather(value=True, root=0)
# REDUCE HERE
if myRank == masterRank:
# Open result files for reading
resultFiles = { }
for i in range(nProcs):
resultFiles[i] = open(tmpdir+'/results.'+str(i),'r')
if FLAGS.train:
##########################################################################
# Compute f-measure over all alignments
##########################################################################
numCorrect = 0
numModelLinks = 0
numGoldLinks = 0
for i, instanceID in enumerate(indices[:FLAGS.subset]):
# What node stored instance i
node = i % nProcs
# Retrieve result from instance i
resultTuple = cPickle.load(resultFiles[node])
modelBest = resultTuple[0]
gold = resultTuple[1]
# Update F-score counts
numCorrect_, numModelLinks_, numGoldLinks_ = f1accumulator(modelBest,
gold)
numCorrect += numCorrect_
numModelLinks += numModelLinks_
numGoldLinks += numGoldLinks_
# Compute F-measure, Precision, and Recall
fmeasure, precision, recall = f1score(numCorrect,
numModelLinks,
numGoldLinks)
elapsedTime = time.time() - startTime
######################################################################
# Print report for this iteration
######################################################################
sys.stderr.write("Time: "+str(elapsedTime)+"\n")
sys.stderr.write("\n")
sys.stderr.write('F-score-%s: %1.5f\n' % (name, fmeasure))
sys.stderr.write('Precision-%s: %1.5f\n' % (name, precision))
sys.stderr.write('Recall-%s: %1.5f\n' % (name, recall))
sys.stderr.write('# Correct: %d\n' % (numCorrect))
sys.stderr.write('# Me Total: %d\n' % (numModelLinks))
sys.stderr.write('# Gold Total: %d\n' % (numGoldLinks))
sys.stderr.write("[%d] Finished decoding.\n" %(myRank))
else:
for i, instanceID in enumerate(indices):
node = i % nProcs
modelBestLinks = cPickle.load(resultFiles[node])
out.write("%s\n" %(" ".join(map(lambda link: "%s-%s" %(link[0], link[1]), modelBestLinks))))
# CLEAN UP
for i in range(nProcs):
resultFiles[i].close()
return
'''
Created on Nov 11, 2010
@author: joel
'''
import boost.mpi as mpi
import numpy as np
from time import time
N = 1000000
#a = np.arange(N, dtype=float)
a = [np.arange(10) for i in range(N/10)]
if mpi.world.rank == 0:
ts = []
ts.append(time())
mpi.broadcast(mpi.world, a, root=0)
ts.append(time())
g = mpi.gather(mpi.world, a, root=0)
ts.append(time())
ta = np.array(ts)
print "Times taken: ",(ta[1:] - ta[:-1])
else:
mpi.broadcast(mpi.world, a, root=0)
mpi.gather(mpi.world, a, root=0)
