out.add("red_" + k, "red_" + v) tserver.run_server(data_gen) inputs = ["ape", "cat", "dog"] params = {"test1": "1,2,3",\ "one two three": "dim\ndam\n",\ "dummy": "value"} job = Disco(sys.argv[1]).new_job( name = "test_external", input = tserver.makeurl(inputs), map = external(["ext_test"]), reduce = fun_reduce, ext_params = params, nr_reduces = 1, sort = False) results = sorted([(v, k) for k, v in result_iterator(job.wait())]) for i, e in enumerate(results): v, k = e if k != "red_dkey" or v != "red_test_%s" % inputs[i / 3]: raise Exception("Invalid answer: %s, %s" % (k, v)) if len(results) != 9: raise Exception("Wrong number of results: %u vs. 9" % len(results)) job.purge() print "ok"
def fun_map(e, params): msg(e) return [] def fun_map2(e, params): return [] tserver.run_server(data_gen) inputs = tserver.makeurl([1]) job = Disco(sys.argv[1]).new_job(name = "test_ratelimit", input = inputs, map = fun_map) time.sleep(5) check_dead(job) job = Disco(sys.argv[1]).new_job(name = "test_ratelimit2", input = inputs, map = fun_map2, status_interval = 1) time.sleep(5) check_dead(job) job = Disco(sys.argv[1]).new_job(name = "test_ratelimit3", input = inputs, map = fun_map2, status_interval = 0) job.wait() job.purge() print "ok"
return 0 def data_gen(path): return "\n".join([path[1:]] * 10) def fun_map(e, params): return [(e, params.f1(int(e), params.x))] def fun_reduce(iter, out, params): for k, v in iter: out.add(k, params.f2(int(v))) tserver.run_server(data_gen) inputs = range(10) job = Disco(sys.argv[1]).new_job(name = "test_params", input = tserver.makeurl(inputs), map = fun_map, params = Params(x = 5, f1 = fun1, f2 = fun2), reduce = fun_reduce, nr_reduces = 1, sort = False) for x, y in result_iterator(job.wait()): if fun2(int(x) + 5) != int(y): raise "Invalid result: %s and %s" % (x, y) job.purge() print "ok"
s = 1 for k, v in iter: if k != "=" + v: raise Exception("Corrupted key") s *= int(v) out.add("result", s) tserver.run_server(data_gen) inputs = [3, 5, 7, 11, 13, 17, 19, 23, 29, 31] job = Disco(sys.argv[1]).new_job( name = "test_writers", input = tserver.makeurl(inputs), map = fun_map, map_writer = fun_map_writer, reduce = fun_reduce, reduce_reader = fun_reduce_reader, reduce_writer = fun_reduce_writer, nr_reduces = 1, sort = False) res = list(result_iterator(job.wait(), reader = result_reader)) if res != [ANS]: raise Exception("Invalid answer: %s" % res) job.purge() print "ok"
tserver.run_server(data_gen) inputs = ["01/11/1965", "14/03/1983", "12/12/2002"] job = Disco(sys.argv[1]).new_job(name = "test_objectrw", input = tserver.makeurl(inputs), map = fun_map, map_writer = func.object_writer, reduce = fun_reduce, reduce_reader = func.object_reader, reduce_writer = func.object_writer, required_modules = ["math", "datetime", "time"], nr_reduces = 1, sort = False) i = 0 for k, v in result_iterator(job.wait(), reader = func.object_reader): if k["PI2"] != math.pi: raise "Invalid key: %s" % k if v.strftime("%d/%m/%Y") not in inputs: raise "Invalid value: %s" % v i += 1 if i != 30: raise "Wrong number of results, got %d, expected 30" % i job.purge() print "ok"
out.add(k, v) tserver.run_server(data_gen) job = Disco(sys.argv[1]).new_job(\ name = "test_profile",\ input = tserver.makeurl([""] * int(100)),\ map = really_unique_function_name,\ reduce = fun_reduce,\ nr_reduces = 30,\ sort = False,\ profile = True) ANS = {"gutta": int(1e4), "cavat": int(2e4), "capidem": int(1e4)} i = 0 for key, value in result_iterator(job.wait()): i += 1 if ANS[key] == int(value): print "Correct: %s %s" % (key, value) else: raise "Results don't match (%s): Got %d expected %d" %\ (key, int(value), ANS[key]) if i != 3: raise "Too few results" buf = cStringIO.StringIO() sys.stdout = buf job.profile_stats().print_stats() sys.stdout = sys.__stdout__ #stats = job.profile_stats()