def run(filename): signal.signal(signal.SIGQUIT, sig_handler) os.system('rm ./_worker_profiles/*') mod_name, _ = splitext(basename(filename)) module = imp.load_source(mod_name, filename) util.log_info('Running benchmarks for module: %s (%s)', module, filename) benchmarks = [ k for k in dir(module) if (k.startswith('benchmark_') and isinstance(getattr(module, k), types.FunctionType)) ] spartan.config.parse(sys.argv) if benchmarks: # csv header print 'num_workers,bench,time' workers = [int(w) for w in FLAGS.worker_list.split(',')] for i in workers: # restart the cluster FLAGS.num_workers = i ctx = spartan.initialize() timer = BenchTimer(i) util.log_info('Running benchmarks on %d workers', i) if FLAGS.test_optimizations: timer.prefix = 'opt_enabled' FLAGS.optimization = 1 run_benchmarks(module, benchmarks, ctx, timer) timer.prefix = 'opt_disabled' FLAGS.optimization = 1 run_benchmarks(module, benchmarks, ctx, timer) spartan.shutdown() time.sleep(1) if FLAGS.profile_worker: util.log_info('Writing worker profiles...') join_profiles('./_worker_profiles')
def run(filename): signal.signal(signal.SIGQUIT, sig_handler) os.system('rm ./_worker_profiles/*') mod_name, _ = splitext(basename(filename)) module = imp.load_source(mod_name, filename) util.log_info('Running benchmarks for module: %s (%s)', module, filename) benchmarks = [k for k in dir(module) if ( k.startswith('benchmark_') and isinstance(getattr(module, k), types.FunctionType)) ] spartan.config.parse(sys.argv) if benchmarks: # csv header print 'num_workers,bench,time' workers = [int(w) for w in FLAGS.worker_list.split(',')] for i in workers: # restart the cluster FLAGS.num_workers = i ctx = spartan.initialize() timer = BenchTimer(i) util.log_info('Running benchmarks on %d workers', i) if FLAGS.test_optimizations: timer.prefix = 'opt_enabled' FLAGS.optimization = 1 run_benchmarks(module, benchmarks, ctx, timer) timer.prefix = 'opt_disabled' FLAGS.optimization = 0 run_benchmarks(module, benchmarks, ctx, timer) spartan.shutdown() time.sleep(1) if FLAGS.profile_worker: util.log_info('Writing worker profiles...') join_profiles('./_worker_profiles')
def test_fn(): ctx = spartan.initialize() fn(ctx)
def setUpClass(cls): cls.ctx = spartan.initialize()
def main(): sp.initialize() l1 = 784 l2 = 300 l3 = 10 # W = (sp.randn(l1, l2) * math.sqrt(4.0 / (l1 + l2))).evaluate() # 784 * 300 matrix W = (sp.randn(l2, l1) * math.sqrt(4.0 / (l1 + l2))).evaluate() # 300 * 784 matrix # V = (sp.randn(l2, l3) * math.sqrt(4.0 / (l2 + l3))).evaluate() # 300 * 10 matrix V = (sp.randn(l3, l2) * math.sqrt(4.0 / (l2 + l3))).evaluate() # 10 * 300 matrix print "V:", V.glom() print "W:", W.glom() # generate distributed matrix rows = 81000 cols = 784 num_workers = 100 subrow = rows / num_workers density = 0.23588 eps_w = 0.01 length = int(subrow * cols * density) + 1 a = np.random.rand(length) b = np.random.rand(length) c = np.random.rand(length) list = [] for i in range(num_workers): r = i % 3 if r == 0: list.append((subrow, cols, a)) elif r == 1: list.append((subrow, cols, b)) else: list.append((subrow, cols, c)) print "starts to generate train_data!" start = datetime.now() train_data = map(_split_sample, list) end = datetime.now() diff = end - start t = diff.microseconds / 1000000.0 + diff.seconds print "generating all matrix spent time: ", t, "secs" print "train data length: ", len(train_data) np.random.shuffle(train_data) count = 0 for (mb_samples, mb_labels) in train_data: begin = time.time() num_samples = mb_samples.shape[0] # input = mb_samples # sample * 784 matrix input = mb_samples.T # 784 * sample matrix # if count > 20 or count == 0: # print "input data:", input.glom() # label = mb_labels # sample * 10 matrix label = mb_labels.T # 10 * sample matrix # ff ffs = time.time() # wIn = sp.dot(input, W) # sample * 300 matrix wIn = sp.dot(W, input) # 300 * sample matrix wOut = sigmoid(wIn) print "V", V # vIn = sp.dot(wOut, V) # sample * 10 matrix vIn = sp.dot(V, wOut) # 10 * sample matrix vOut = sigmoid(vIn) print "W.shape:%s, input shape:%s, V.shpae:%s, wOut.shpae:%s" % (str( W.shape), str(input.shape), str(V.shape), str(wOut.shape)) vOut.evaluate() ffe = time.time() #bp bps = time.time() # o = vOut - label # if count > 20 or count == 0: # print "vOut - label", o.glom() # vDelta = dsigmoid(vIn) * o # sample * 10 matrix vDelta = dsigmoid(vIn) * (vOut - label) # 10 * sample matrix # wDelta = dsigmoid(wIn) * (sp.dot(vDelta, V.T)) # sample * 300 matrix wDelta = dsigmoid(wIn) * (sp.dot(V.T, vDelta)) # 300 * sample matrix # wDelta.evaluate() bpe = time.time() # update upb = time.time() # V -= eps_w * sp.dot(wOut.T, vDelta) / num_samples # 300 * 10 matrix V -= eps_w * sp.dot(vDelta, wOut.T) / num_samples # 10 * 300 matrix # W -= eps_w * sp.dot(input.T, wDelta) / num_samples # 784 * 300 matrix W -= eps_w * sp.dot(wDelta, input.T) / num_samples # 300 * 784 matrix V.evaluate() W.evaluate() count = count + 1 # if count > 20: # print "V:", V.glom() # print "W:", W.glom() upe = time.time() print "ff used time: %f, bp used time: %f, update used time: %f, num_samples: %d \n" % ( ffe - ffs, bpe - bps, upe - upb, num_samples) diff = time.time() - begin print "iteration: %d, spent time: %f\n" % (count, diff)
tile_hint = (dim / ctx.num_workers, 1) ), make_dist, ) ) linkMatrix = eager( expr.shuffle( expr.ndarray( (dim, dim), dtype = np.int64, tile_hint = (dim, dim / ctx.num_workers)), make_matrix, )) startCompute = time.time() for it in range(numIters): first = expr.add(dist, linkMatrix) second = first.min(axis = 0) dist = second.reshape(dim, 1) dist.evaluate() endCompute = time.time() return endCompute - startCompute if __name__ == "__main__": ctx = sp.initialize() filePath = r"/home/wangpeng/projects/spartan-study/test" numIters = 10 dim = 5000 computeTime = shortestPath(ctx, dim, numIters) print "Cost time: %s" % computeTime
import time import spartan as sp import sys from datetime import datetime #argvLen = len(sys.argv); #print "total fileds in sys.argv=",argvLen; #for i,eachArg in enumerate(sys.argv): # print "[%d]=%s"%(i, eachArg); m = int(sys.argv[1]) k = int(sys.argv[2]) n = int(sys.argv[3]) sp.initialize() print "execute matrix multiplication, m: %d, k: %d, n: %d. date: %s\n" % ( m, k, n, datetime.now()) a = sp.rand(m, k) b = sp.rand(k, n) start = time.time() z = sp.dot(a, b).optimized() z.evaluate() end = time.time() print 'z: ', z print "finish matrix multiplication, diff time: %d. date: %s\n" % ( (end - start), datetime.now())
import spartan as sp from datetime import datetime import numpy as np import scipy.sparse import multiprocessing import time wrows = vrows = 18000 hcols = vcols = 480000 wcols = hrows = 200 sp.initialize() V = sp.sparse_rand((vrows, vcols), density=0.01, dtype=np.double, format=u'csr') W = sp.rand(wrows, wcols) H = sp.rand(hrows, hcols) eps = 10e-8 max_iteration = 5 i = 0 print "starts to run!" start = time.time() while i < max_iteration: begin = time.time() H = H * (sp.dot(W.T, V) / (sp.dot(sp.dot(W.T, W), H) + eps)) W = W * (sp.dot(V, H.T) / (sp.dot(W, sp.dot(H, H.T)) + eps)) i = i + 1 end = time.time()
[0, 1/2.0, 0, 0, 0, 0], [0, 1/2.0, 1/3.0, 0, 0, 0], [0, 0, 1/3.0, 0, 0, 0], [1/2.0, 0, 1/3.0, 0, 1, 0]], dtype = "float32") alpha = 0.15 #n = np.shape(A)[1] #print n #initRank = [1] * n filePath = "graph" linkMatrix = loadMatrix(filePath) initRank = np.ones(4847571, dtype = "float32") numPage = 4847571 numIters = 10 start = time.time() sr = pagerankDistributed(page, numIters, alpha, linkMatrix, initRank) end = time.time() print "Cost time: %s" % (end - start) #nr = pagerank(A, initRank, m, numIters) #Assert.all_eq(nr, sr.glom()) if __name__ == '__main__': sAllTime = time.time() ctx = spartan.initialize() alpha = 0.15 numPage = 4000000 numIters = 10 generateTime, computeTime = pagerankDistributed(ctx, numPage, numIters, alpha) eAllTime = time.time() info = "Generate time: %s; Compute time: %s; Total time: %s" % (generateTime, computeTime, eAllTime - sAllTime) util.log_info(info)
self.b2.evaluate() #if count % 40 == 0 or iterations == 0: if count % 40 == 0: #correct = np.argmax(out, axis=0) - np.argmax(target, axis=0) #print 'Training error:', float(np.count_nonzero(correct)) / num_samples correct = spartan.argmax(out, axis=0) - spartan.argmax(target, axis=0) print 'Training error:', (float(spartan.count_nonzero(correct).glom()) / num_samples) count = count + 1 print 'spent ', (time.time() - begin) # test #a1 = test_samples.T #a2 = relu(np.dot(self.w1, a1) + self.b1) #a3 = np.dot(self.w2, a2) + self.b2 #correct = np.argmax(a3, axis=0) - np.argmax(test_labels.T, axis=0) a1 = test_samples.T a2 = relu(spartan.dot(self.w1, a1) + self.b1) a3 = spartan.dot(self.w2, a2) + self.b2 correct = spartan.argmax(a3, axis=0) - spartan.argmax(test_labels.T, axis=0) ##print correct #print 'Testing error:', float(np.count_nonzero(correct)) / num_test_samples #print '---Finish epoch #%d' % epoch if __name__ == '__main__': spartan.config.parse(sys.argv) ctx = spartan.initialize() trainer = MnistTrainer(num_epochs=100, ctx=ctx, mb_size=256 * 2 * ctx.num_workers) trainer.run()