def test_write_to_stdout(self): worker_1 = RemoteWorker('localhost') master = Master([worker_1]) dataset = DataSet(range(0, 100)) result = master.run(write_to_stdout, params=1, dataset=dataset) assert_equals(42, result)
def test_single_worker_multiply_sum_simple(self): worker = RemoteWorker('localhost') master = Master([worker]) dataset = DataSet(range(0,100)) result = master.run(multiply_sum_simple, params=3, dataset=dataset) assert_equals(14850, result)
def test_single_worker_simple(self): worker = RemoteWorker('localhost') master = Master([worker]) assert_equals(42, master.run(constant)) params = [1, 2, 3, 6, 7, 9] assert_equals(sum(params), master.run(sum_params, params))
def test_dual_worker_simple(self): worker_1 = RemoteWorker('localhost') worker_2 = RemoteWorker('localhost') master = Master([worker_1, worker_2]) assert_equals(84, master.run(constant)) params = [1, 2, 3, 6, 7, 9] assert_equals(2 * sum(params), master.run(sum_params, params))
def test_numpy(self): worker_1 = RemoteWorker('localhost') worker_2 = RemoteWorker('localhost') master = Master([worker_1, worker_2]) dataset = DataSet(numpy.array([[1,2],[3,4],[5,6],[7,8],[9,10]])) params = numpy.array([2,4]) result = master.run(inner_prod, params=params, dataset=dataset) assert_equals(170, result)
def main(): _fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s' logging.basicConfig(level=logging.INFO, format=_fmt) # Worker hostnames are the commandline options. Each hostname specified on # the commandline will be bootstrapped and have a worker started on it. if len(sys.argv) < 2: print 'usage: %s <matrixsize> [<worker> ...]' % sys.argv[0] sys.exit(1) matsize = int(sys.argv[1]) hostnames = sys.argv[2:] or ['localhost'] # Start the worker processes on the specified hostnames. workers = [] for i, hostname in enumerate(hostnames): worker = RemoteWorker(hostname) workers.append(worker) print 'Started worker %d on %s' % (i, hostname) print # Create a numpy matrix. X = numpy.random.permutation(numpy.arange(1,matsize*matsize+1)) X = X.reshape(matsize, matsize) print 'Creating and permuting %dx%d matrix X with integers 1 .. %d^2' % \ (matsize, matsize, matsize) numpy.set_printoptions(threshold=256) print 'X = %s' % (str(X).replace("\n", "\n ")) exp_sum = (1+matsize**2)*(matsize**2)/2 print 'Expected element sum of X = %d = (1/2)(%d^2+1)(%d^2)' % \ (exp_sum, matsize, matsize) # Create a DataSet object from this matrix. # # DataSet objects are necessary so the framework can determine how to split # up the data into smaller chunks and distribute the chunks to the workers. dataset = NumpyMatrixDataSet(X) # Sum the matrix! master = Master(workers) params = None # No parameters needed for this job. result = master.run(sum_dataset, params, dataset) # Print the result. print 'Computed element sum of X = %d' % result