def test_write_to_stdout(self):
worker_1 = RemoteWorker('localhost')
master = Master([worker_1])
dataset = DataSet(range(0, 100))
result = master.run(write_to_stdout, params=1, dataset=dataset)
assert_equals(42, result)
def test_single_worker_multiply_sum_simple(self):
worker = RemoteWorker('localhost')
master = Master([worker])
dataset = DataSet(range(0,100))
result = master.run(multiply_sum_simple, params=3, dataset=dataset)
assert_equals(14850, result)
def test_single_worker_simple(self):
worker = RemoteWorker('localhost')
master = Master([worker])
assert_equals(42, master.run(constant))
params = [1, 2, 3, 6, 7, 9]
assert_equals(sum(params), master.run(sum_params, params))
def test_dual_worker_simple(self):
worker_1 = RemoteWorker('localhost')
worker_2 = RemoteWorker('localhost')
master = Master([worker_1, worker_2])
assert_equals(84, master.run(constant))
params = [1, 2, 3, 6, 7, 9]
assert_equals(2 * sum(params), master.run(sum_params, params))
def test_numpy(self):
worker_1 = RemoteWorker('localhost')
worker_2 = RemoteWorker('localhost')
master = Master([worker_1, worker_2])
dataset = DataSet(numpy.array([[1,2],[3,4],[5,6],[7,8],[9,10]]))
params = numpy.array([2,4])
result = master.run(inner_prod, params=params, dataset=dataset)
assert_equals(170, result)
def main():
_fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=_fmt)
# Worker hostnames are the commandline options. Each hostname specified on
# the commandline will be bootstrapped and have a worker started on it.
if len(sys.argv) < 2:
print 'usage: %s <matrixsize> [<worker> ...]' % sys.argv[0]
sys.exit(1)
matsize = int(sys.argv[1])
hostnames = sys.argv[2:] or ['localhost']
# Start the worker processes on the specified hostnames.
workers = []
for i, hostname in enumerate(hostnames):
worker = RemoteWorker(hostname)
workers.append(worker)
print 'Started worker %d on %s' % (i, hostname)
print
# Create a numpy matrix.
X = numpy.random.permutation(numpy.arange(1,matsize*matsize+1))
X = X.reshape(matsize, matsize)
print 'Creating and permuting %dx%d matrix X with integers 1 .. %d^2' % \
(matsize, matsize, matsize)
numpy.set_printoptions(threshold=256)
print 'X = %s' % (str(X).replace("\n", "\n "))
exp_sum = (1+matsize**2)*(matsize**2)/2
print 'Expected element sum of X = %d = (1/2)(%d^2+1)(%d^2)' % \
(exp_sum, matsize, matsize)
# Create a DataSet object from this matrix.
#
# DataSet objects are necessary so the framework can determine how to split
# up the data into smaller chunks and distribute the chunks to the workers.
dataset = NumpyMatrixDataSet(X)
# Sum the matrix!
master = Master(workers)
params = None # No parameters needed for this job.
result = master.run(sum_dataset, params, dataset)
# Print the result.
print 'Computed element sum of X = %d' % result
