def run(filename):
signal.signal(signal.SIGQUIT, sig_handler)
os.system('rm ./_worker_profiles/*')
mod_name, _ = splitext(basename(filename))
module = imp.load_source(mod_name, filename)
util.log_info('Running benchmarks for module: %s (%s)', module, filename)
benchmarks = [
k for k in dir(module)
if (k.startswith('benchmark_')
and isinstance(getattr(module, k), types.FunctionType))
]
spartan.config.parse(sys.argv)
if benchmarks:
# csv header
print 'num_workers,bench,time'
workers = [int(w) for w in FLAGS.worker_list.split(',')]
for i in workers:
# restart the cluster
FLAGS.num_workers = i
ctx = spartan.initialize()
timer = BenchTimer(i)
util.log_info('Running benchmarks on %d workers', i)
if FLAGS.test_optimizations:
timer.prefix = 'opt_enabled'
FLAGS.optimization = 1
run_benchmarks(module, benchmarks, ctx, timer)
timer.prefix = 'opt_disabled'
FLAGS.optimization = 1
run_benchmarks(module, benchmarks, ctx, timer)
spartan.shutdown()
time.sleep(1)
if FLAGS.profile_worker:
util.log_info('Writing worker profiles...')
join_profiles('./_worker_profiles')
def run(filename):
signal.signal(signal.SIGQUIT, sig_handler)
os.system('rm ./_worker_profiles/*')
mod_name, _ = splitext(basename(filename))
module = imp.load_source(mod_name, filename)
util.log_info('Running benchmarks for module: %s (%s)', module, filename)
benchmarks = [k for k in dir(module) if (
k.startswith('benchmark_') and
isinstance(getattr(module, k), types.FunctionType))
]
spartan.config.parse(sys.argv)
if benchmarks:
# csv header
print 'num_workers,bench,time'
workers = [int(w) for w in FLAGS.worker_list.split(',')]
for i in workers:
# restart the cluster
FLAGS.num_workers = i
ctx = spartan.initialize()
timer = BenchTimer(i)
util.log_info('Running benchmarks on %d workers', i)
if FLAGS.test_optimizations:
timer.prefix = 'opt_enabled'
FLAGS.optimization = 1
run_benchmarks(module, benchmarks, ctx, timer)
timer.prefix = 'opt_disabled'
FLAGS.optimization = 0
run_benchmarks(module, benchmarks, ctx, timer)
spartan.shutdown()
time.sleep(1)
if FLAGS.profile_worker:
util.log_info('Writing worker profiles...')
join_profiles('./_worker_profiles')
def test_fn():
ctx = spartan.initialize()
fn(ctx)
def setUpClass(cls): cls.ctx = spartan.initialize()
def main():
sp.initialize()
l1 = 784
l2 = 300
l3 = 10
# W = (sp.randn(l1, l2) * math.sqrt(4.0 / (l1 + l2))).evaluate() # 784 * 300 matrix
W = (sp.randn(l2, l1) *
math.sqrt(4.0 / (l1 + l2))).evaluate() # 300 * 784 matrix
# V = (sp.randn(l2, l3) * math.sqrt(4.0 / (l2 + l3))).evaluate() # 300 * 10 matrix
V = (sp.randn(l3, l2) * math.sqrt(4.0 /
(l2 + l3))).evaluate() # 10 * 300 matrix
print "V:", V.glom()
print "W:", W.glom()
# generate distributed matrix
rows = 81000
cols = 784
num_workers = 100
subrow = rows / num_workers
density = 0.23588
eps_w = 0.01
length = int(subrow * cols * density) + 1
a = np.random.rand(length)
b = np.random.rand(length)
c = np.random.rand(length)
list = []
for i in range(num_workers):
r = i % 3
if r == 0:
list.append((subrow, cols, a))
elif r == 1:
list.append((subrow, cols, b))
else:
list.append((subrow, cols, c))
print "starts to generate train_data!"
start = datetime.now()
train_data = map(_split_sample, list)
end = datetime.now()
diff = end - start
t = diff.microseconds / 1000000.0 + diff.seconds
print "generating all matrix spent time: ", t, "secs"
print "train data length: ", len(train_data)
np.random.shuffle(train_data)
count = 0
for (mb_samples, mb_labels) in train_data:
begin = time.time()
num_samples = mb_samples.shape[0]
# input = mb_samples # sample * 784 matrix
input = mb_samples.T # 784 * sample matrix
# if count > 20 or count == 0:
# print "input data:", input.glom()
# label = mb_labels # sample * 10 matrix
label = mb_labels.T # 10 * sample matrix
# ff
ffs = time.time()
# wIn = sp.dot(input, W) # sample * 300 matrix
wIn = sp.dot(W, input) # 300 * sample matrix
wOut = sigmoid(wIn)
print "V", V
# vIn = sp.dot(wOut, V) # sample * 10 matrix
vIn = sp.dot(V, wOut) # 10 * sample matrix
vOut = sigmoid(vIn)
print "W.shape:%s, input shape:%s, V.shpae:%s, wOut.shpae:%s" % (str(
W.shape), str(input.shape), str(V.shape), str(wOut.shape))
vOut.evaluate()
ffe = time.time()
#bp
bps = time.time()
# o = vOut - label
# if count > 20 or count == 0:
# print "vOut - label", o.glom()
# vDelta = dsigmoid(vIn) * o # sample * 10 matrix
vDelta = dsigmoid(vIn) * (vOut - label) # 10 * sample matrix
# wDelta = dsigmoid(wIn) * (sp.dot(vDelta, V.T)) # sample * 300 matrix
wDelta = dsigmoid(wIn) * (sp.dot(V.T, vDelta)) # 300 * sample matrix
# wDelta.evaluate()
bpe = time.time()
# update
upb = time.time()
# V -= eps_w * sp.dot(wOut.T, vDelta) / num_samples # 300 * 10 matrix
V -= eps_w * sp.dot(vDelta, wOut.T) / num_samples # 10 * 300 matrix
# W -= eps_w * sp.dot(input.T, wDelta) / num_samples # 784 * 300 matrix
W -= eps_w * sp.dot(wDelta, input.T) / num_samples # 300 * 784 matrix
V.evaluate()
W.evaluate()
count = count + 1
# if count > 20:
# print "V:", V.glom()
# print "W:", W.glom()
upe = time.time()
print "ff used time: %f, bp used time: %f, update used time: %f, num_samples: %d \n" % (
ffe - ffs, bpe - bps, upe - upb, num_samples)
diff = time.time() - begin
print "iteration: %d, spent time: %f\n" % (count, diff)
tile_hint = (dim / ctx.num_workers, 1) ), make_dist, ) ) linkMatrix = eager( expr.shuffle( expr.ndarray( (dim, dim), dtype = np.int64, tile_hint = (dim, dim / ctx.num_workers)), make_matrix, )) startCompute = time.time() for it in range(numIters): first = expr.add(dist, linkMatrix) second = first.min(axis = 0) dist = second.reshape(dim, 1) dist.evaluate() endCompute = time.time() return endCompute - startCompute if __name__ == "__main__": ctx = sp.initialize() filePath = r"/home/wangpeng/projects/spartan-study/test" numIters = 10 dim = 5000 computeTime = shortestPath(ctx, dim, numIters) print "Cost time: %s" % computeTime
import time
import spartan as sp
import sys
from datetime import datetime
#argvLen = len(sys.argv);
#print "total fileds in sys.argv=",argvLen;
#for i,eachArg in enumerate(sys.argv):
# print "[%d]=%s"%(i, eachArg);
m = int(sys.argv[1])
k = int(sys.argv[2])
n = int(sys.argv[3])
sp.initialize()
print "execute matrix multiplication, m: %d, k: %d, n: %d. date: %s\n" % (
m, k, n, datetime.now())
a = sp.rand(m, k)
b = sp.rand(k, n)
start = time.time()
z = sp.dot(a, b).optimized()
z.evaluate()
end = time.time()
print 'z: ', z
print "finish matrix multiplication, diff time: %d. date: %s\n" % (
(end - start), datetime.now())
import spartan as sp
from datetime import datetime
import numpy as np
import scipy.sparse
import multiprocessing
import time
wrows = vrows = 18000
hcols = vcols = 480000
wcols = hrows = 200
sp.initialize()
V = sp.sparse_rand((vrows, vcols), density=0.01, dtype=np.double, format=u'csr')
W = sp.rand(wrows, wcols)
H = sp.rand(hrows, hcols)
eps = 10e-8
max_iteration = 5
i = 0
print "starts to run!"
start = time.time()
while i < max_iteration:
begin = time.time()
H = H * (sp.dot(W.T, V) / (sp.dot(sp.dot(W.T, W), H) + eps))
W = W * (sp.dot(V, H.T) / (sp.dot(W, sp.dot(H, H.T)) + eps))
i = i + 1
end = time.time()
[0, 1/2.0, 0, 0, 0, 0], [0, 1/2.0, 1/3.0, 0, 0, 0], [0, 0, 1/3.0, 0, 0, 0], [1/2.0, 0, 1/3.0, 0, 1, 0]], dtype = "float32") alpha = 0.15 #n = np.shape(A)[1] #print n #initRank = [1] * n filePath = "graph" linkMatrix = loadMatrix(filePath) initRank = np.ones(4847571, dtype = "float32") numPage = 4847571 numIters = 10 start = time.time() sr = pagerankDistributed(page, numIters, alpha, linkMatrix, initRank) end = time.time() print "Cost time: %s" % (end - start) #nr = pagerank(A, initRank, m, numIters) #Assert.all_eq(nr, sr.glom()) if __name__ == '__main__': sAllTime = time.time() ctx = spartan.initialize() alpha = 0.15 numPage = 4000000 numIters = 10 generateTime, computeTime = pagerankDistributed(ctx, numPage, numIters, alpha) eAllTime = time.time() info = "Generate time: %s; Compute time: %s; Total time: %s" % (generateTime, computeTime, eAllTime - sAllTime) util.log_info(info)
self.b2.evaluate()
#if count % 40 == 0 or iterations == 0:
if count % 40 == 0:
#correct = np.argmax(out, axis=0) - np.argmax(target, axis=0)
#print 'Training error:', float(np.count_nonzero(correct)) / num_samples
correct = spartan.argmax(out, axis=0) - spartan.argmax(target, axis=0)
print 'Training error:', (float(spartan.count_nonzero(correct).glom())
/ num_samples)
count = count + 1
print 'spent ', (time.time() - begin)
# test
#a1 = test_samples.T
#a2 = relu(np.dot(self.w1, a1) + self.b1)
#a3 = np.dot(self.w2, a2) + self.b2
#correct = np.argmax(a3, axis=0) - np.argmax(test_labels.T, axis=0)
a1 = test_samples.T
a2 = relu(spartan.dot(self.w1, a1) + self.b1)
a3 = spartan.dot(self.w2, a2) + self.b2
correct = spartan.argmax(a3, axis=0) - spartan.argmax(test_labels.T, axis=0)
##print correct
#print 'Testing error:', float(np.count_nonzero(correct)) / num_test_samples
#print '---Finish epoch #%d' % epoch
if __name__ == '__main__':
spartan.config.parse(sys.argv)
ctx = spartan.initialize()
trainer = MnistTrainer(num_epochs=100, ctx=ctx, mb_size=256 * 2 * ctx.num_workers)
trainer.run()
def test_fn():
ctx = spartan.initialize()
fn(ctx)
def setUpClass(cls):
cls.ctx = spartan.initialize()
