out.add("red_" + k, "red_" + v)
tserver.run_server(data_gen)
inputs = ["ape", "cat", "dog"]
params = {"test1": "1,2,3",\
"one two three": "dim\ndam\n",\
"dummy": "value"}
job = Disco(sys.argv[1]).new_job(
name = "test_external",
input = tserver.makeurl(inputs),
map = external(["ext_test"]),
reduce = fun_reduce,
ext_params = params,
nr_reduces = 1,
sort = False)
results = sorted([(v, k) for k, v in result_iterator(job.wait())])
for i, e in enumerate(results):
v, k = e
if k != "red_dkey" or v != "red_test_%s" % inputs[i / 3]:
raise Exception("Invalid answer: %s, %s" % (k, v))
if len(results) != 9:
raise Exception("Wrong number of results: %u vs. 9" % len(results))
job.purge()
print "ok"
def fun_map(e, params):
msg(e)
return []
def fun_map2(e, params):
return []
tserver.run_server(data_gen)
inputs = tserver.makeurl([1])
job = Disco(sys.argv[1]).new_job(name = "test_ratelimit",
input = inputs, map = fun_map)
time.sleep(5)
check_dead(job)
job = Disco(sys.argv[1]).new_job(name = "test_ratelimit2",
input = inputs, map = fun_map2, status_interval = 1)
time.sleep(5)
check_dead(job)
job = Disco(sys.argv[1]).new_job(name = "test_ratelimit3",
input = inputs, map = fun_map2, status_interval = 0)
job.wait()
job.purge()
print "ok"
return 0
def data_gen(path):
return "\n".join([path[1:]] * 10)
def fun_map(e, params):
return [(e, params.f1(int(e), params.x))]
def fun_reduce(iter, out, params):
for k, v in iter:
out.add(k, params.f2(int(v)))
tserver.run_server(data_gen)
inputs = range(10)
job = Disco(sys.argv[1]).new_job(name = "test_params",
input = tserver.makeurl(inputs),
map = fun_map,
params = Params(x = 5, f1 = fun1, f2 = fun2),
reduce = fun_reduce,
nr_reduces = 1,
sort = False)
for x, y in result_iterator(job.wait()):
if fun2(int(x) + 5) != int(y):
raise "Invalid result: %s and %s" % (x, y)
job.purge()
print "ok"
s = 1
for k, v in iter:
if k != "=" + v:
raise Exception("Corrupted key")
s *= int(v)
out.add("result", s)
tserver.run_server(data_gen)
inputs = [3, 5, 7, 11, 13, 17, 19, 23, 29, 31]
job = Disco(sys.argv[1]).new_job(
name = "test_writers",
input = tserver.makeurl(inputs),
map = fun_map,
map_writer = fun_map_writer,
reduce = fun_reduce,
reduce_reader = fun_reduce_reader,
reduce_writer = fun_reduce_writer,
nr_reduces = 1,
sort = False)
res = list(result_iterator(job.wait(), reader = result_reader))
if res != [ANS]:
raise Exception("Invalid answer: %s" % res)
job.purge()
print "ok"
tserver.run_server(data_gen)
inputs = ["01/11/1965", "14/03/1983", "12/12/2002"]
job = Disco(sys.argv[1]).new_job(name = "test_objectrw",
input = tserver.makeurl(inputs),
map = fun_map,
map_writer = func.object_writer,
reduce = fun_reduce,
reduce_reader = func.object_reader,
reduce_writer = func.object_writer,
required_modules = ["math", "datetime", "time"],
nr_reduces = 1,
sort = False)
i = 0
for k, v in result_iterator(job.wait(), reader = func.object_reader):
if k["PI2"] != math.pi:
raise "Invalid key: %s" % k
if v.strftime("%d/%m/%Y") not in inputs:
raise "Invalid value: %s" % v
i += 1
if i != 30:
raise "Wrong number of results, got %d, expected 30" % i
job.purge()
print "ok"
out.add(k, v)
tserver.run_server(data_gen)
job = Disco(sys.argv[1]).new_job(\
name = "test_profile",\
input = tserver.makeurl([""] * int(100)),\
map = really_unique_function_name,\
reduce = fun_reduce,\
nr_reduces = 30,\
sort = False,\
profile = True)
ANS = {"gutta": int(1e4), "cavat": int(2e4), "capidem": int(1e4)}
i = 0
for key, value in result_iterator(job.wait()):
i += 1
if ANS[key] == int(value):
print "Correct: %s %s" % (key, value)
else:
raise "Results don't match (%s): Got %d expected %d" %\
(key, int(value), ANS[key])
if i != 3:
raise "Too few results"
buf = cStringIO.StringIO()
sys.stdout = buf
job.profile_stats().print_stats()
sys.stdout = sys.__stdout__
#stats = job.profile_stats()
