out.add("red_" + k, "red_" + v)
tserver.run_server(data_gen)
inputs = ["ape", "cat", "dog"]
params = {"test1": "1,2,3",\
"one two three": "dim\ndam\n",\
"dummy": "value"}
job = Disco(sys.argv[1]).new_job(
name = "test_external",
input = tserver.makeurl(inputs),
map = external(["ext_test"]),
reduce = fun_reduce,
ext_params = params,
nr_reduces = 1,
sort = False)
results = sorted([(v, k) for k, v in result_iterator(job.wait())])
for i, e in enumerate(results):
v, k = e
if k != "red_dkey" or v != "red_test_%s" % inputs[i / 3]:
raise Exception("Invalid answer: %s, %s" % (k, v))
if len(results) != 9:
raise Exception("Wrong number of results: %u vs. 9" % len(results))
job.purge()
print "ok"
params={"suffix": "0"},
).wait()
i = 1
while i < 10:
nresults = disco.new_job(
name="test_chain_%d" % i,
input=results,
map=fun_map,
reduce=fun_reduce,
nr_reduces=4,
map_reader=chain_reader,
sort=False,
clean=True,
params={"suffix": str(i)},
).wait()
disco.purge(jobname(results[0]))
results = nresults
i += 1
for key, value in result_iterator(results):
if key[:5] not in ani or key[5:] != "0-1-2-3-4-5-6-7-8-9-":
raise "Corrupted key: %s" % key
if value != "9":
raise "Corrupted value: %s" % value
disco.purge(jobname(results[0]))
print "ok"
