from disco import Disco, result_iterator
def data_gen(path):
return path[1:] + "\n"
def fun_map(e, params):
k = str(int(math.ceil(float(e))) ** 2)
return [(md5.new(k).hexdigest(), "")]
tserver.run_server(data_gen)
disco = Disco(sys.argv[1])
inputs = [1, 485, 3245]
job = disco.new_job(name = "test_reqmodules",
nr_reduces = 1,
input = tserver.makeurl(inputs),
map = fun_map,
required_modules = ["math", "md5"],
sort = False)
res = list(result_iterator(job.wait()))
if len(res) != len(inputs):
raise Exception("Too few results: Got: %d Should be %d" %
(len(res), len(inputs)))
cor = map(lambda x: md5.new(str(int(math.ceil(x)) ** 2)).hexdigest(), inputs)
for k, v in res:
if k not in cor:
raise Exception("Invalid answer: %s" % k)
cor.remove(k)
def fun_reduce(iter, out, params):
for k, v in iter:
out.add("[%s]" % k, v)
tserver.run_server(data_gen)
disco = Disco(sys.argv[1])
num = sum(x['max_workers'] for x in disco.nodeinfo()['available'])
print >> sys.stderr, num, "slots available"
inputs = tserver.makeurl(range(num * 10))
random.shuffle(inputs)
jobs = []
for i in range(5):
jobs.append(disco.new_job(name = "test_async_%d" % i,
input = inputs[i * (num * 2):(i + 1) * (num * 2)],
map = fun_map, reduce = fun_reduce, nr_reduces = 11,
sort = False))
time.sleep(1)
all = dict(("[%s]" % i, 0) for i in range(num * 10))
for job in jobs:
results = job.wait()
print "Job", job, "done"
for k, v in result_iterator(results):
all[k] += 1
job.clean()
for v in all.values():
if v != 10:
raise "Invalid results: %s" % all
N = 100
results = {}
inputs = []
for i in range(N):
a = [i] * 10
b = range(i, i + 10)
inputs += ["%d:%d" % x for x in zip(a, b)]
results[str(i)] = str(sum(b))
disco = Disco(sys.argv[1])
job = disco.new_job(\
name = "test_chunked",
input = tserver.makeurl(inputs),
map = fun_map,
partition = fun_partition,
reduce = fun_reduce,
chunked = False,
nr_reduces = N,
sort = False)
for k, v in result_iterator(job.wait()):
if results[k] != v:
raise "Invalid result, got %s, expected %s" %\
(v, results[k])
del results[k]
if results:
raise "Not enough results"
job.purge()
return [(e + params["suffix"], 0)]
def fun_reduce(iter, out, params):
for k, v in iter:
out.add(k + "-", v)
tserver.run_server(data_gen)
disco = Disco(sys.argv[1])
results = disco.new_job(
name="test_chain_0",
input=tserver.makeurl([""] * 100),
map=fun_map,
reduce=fun_reduce,
nr_reduces=4,
sort=False,
clean=True,
params={"suffix": "0"},
).wait()
i = 1
while i < 10:
nresults = disco.new_job(
name="test_chain_%d" % i,
input=results,
map=fun_map,
reduce=fun_reduce,
nr_reduces=4,
map_reader=chain_reader,
sort=False,
import tserver, sys, time
from disco import Disco
def data_gen(path):
return "1 2 3\n"
def fun_map(e, params):
import time
time.sleep(100)
return []
disco = Disco(sys.argv[1])
num = sum(x['max_workers'] for x in disco.nodeinfo()['available'])
print >> sys.stderr, num, "slots available"
tserver.run_server(data_gen)
job = disco.new_job(name = "test_kill",
input = tserver.makeurl([""] * num * 2), map = fun_map)
time.sleep(10)
print >> sys.stderr, "Killing", job.name
job.kill()
time.sleep(5)
if job.jobinfo()['active'] == "dead":
print "ok"
job.purge()
else:
raise Exception("Killing failed")
out.add("all", "ok")
def fun_map2(e, params):
x = get(e, params.job)
if x != "value:" + e:
raise "Invalid value for key %s: %s" % (e, x)
return [("good", "")]
tserver.run_server(data_gen)
inputs = list(string.ascii_lowercase)
disco = Disco(sys.argv[1])
job1 = disco.new_job(name = "test_oob1",
input = tserver.makeurl(inputs),
map = fun_map,
reduce = fun_reduce,
nr_reduces = 10)
res = list(result_iterator(job1.wait()))
if [("all", "ok")] * 10 != res:
raise "Invalid result: %s" % res
keys = ["reduce:%d" % i for i in range(10)] + inputs
lst = job1.oob_list()
if len(lst) != len(keys):
raise "Invalid number of OOB keys: got %d, expected %d" %\
(len(lst), len(keys))
for key in job1.oob_list():