def test_future_reuse3(loom_env): loom_env.start(3) a = tasks.run("ls") f = loom_env.client.submit_one(a) COUNT = 1000 b = [tasks.run("ls") for i in range(COUNT)] g = loom_env.client.submit(b) result2 = f.fetch() result = loom_env.submit_and_gather(tasks.merge(g)) loom_env.client.release(g) f.release() assert result2 * COUNT == result loom_env.check_final_state()
def test_run_chain(loom_env): const = b"ABC" * 10000000 a = tasks.const(const) b1 = tasks.run(pytestprog(0.01, file_out="test"), stdin=a, outputs=["test"]) b2 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"), inputs=[(b1, "in")], outputs=["test"]) b3 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"), inputs=[(b2, "in")], outputs=["test"]) b4 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"), inputs=[(b3, "in")], outputs=["test"]) loom_env.start(1) result = loom_env.submit_and_gather(b4) assert result == const
def test_reconnect(loom_env): loom_env.start(2) code = """ import sys sys.path.insert(0, "{LOOM_PYTHON}") from loom.client import Client, tasks client = Client("localhost", {PORT}) a = tasks.run("sleep 1") b = tasks.run("sleep 1") c = tasks.run("sleep 1") d = tasks.run("sleep 1") fs = client.submit((a, b, c, d)) print(client.gather(fs)) """.format(LOOM_PYTHON=LOOM_PYTHON, PORT=loom_env.PORT) p = loom_env.independent_python(code) time.sleep(0.6) assert not p.poll() p.kill() p = loom_env.independent_python(code) time.sleep(0.6) assert not p.poll() p.kill() p = loom_env.independent_python(code) time.sleep(0.2) assert not p.poll() p.kill() time.sleep(0.2) a = tasks.run("ls") result = loom_env.submit_and_gather(a) assert result loom_env.check_final_state() a = tasks.const("abc") b = tasks.const("xyz") c = tasks.const("123") d = tasks.merge((a, b, c)) result = loom_env.submit_and_gather(d) assert result == b"abcxyz123" a = tasks.run("ls") result = loom_env.submit_and_gather(a) assert result loom_env.check_final_state()
def test_data_save(loom_env): loom_env.start(1) a1 = tasks.const("ABC" * 10000) a2 = tasks.const(b"") a3 = tasks.run("ls") with pytest.raises(client.TaskFailed): b = tasks.save(a1, "test-file") loom_env.submit_and_gather(b) with pytest.raises(client.TaskFailed): b = tasks.save(a1, "/") loom_env.submit_and_gather(b) b1 = tasks.save(a1, loom_env.get_filename("file1")) b2 = tasks.save(a2, loom_env.get_filename("file2")) b3 = tasks.save(a3, loom_env.get_filename("file3")) result = loom_env.submit_and_gather((b1, b2, b3)) assert result == [b"", b"", b""] with open(loom_env.get_filename("file1")) as f: assert f.read() == "ABC" * 10000 with open(loom_env.get_filename("file2")) as f: assert f.read() == "" with open(loom_env.get_filename("file3")) as f: content = f.read().split("\n") assert "+err" in content assert "+out" in content
def test_run_variable2(loom_env): a = tasks.const("123") b = tasks.const("456") c = tasks.run("/bin/echo $xyz xyz $ab $c", inputs=[(a, "$xyz"), (b, "$c")]) loom_env.start(1) result = loom_env.submit_and_gather(c) assert result == b"123 xyz $ab 456\n"
def test_run_files(loom_env): a1 = tasks.const("abcd" * 100) b1 = tasks.run( pytestprog(0.0, "plus1", file_out="myfile1"), stdin=a1, outputs=["myfile1"]) c1 = tasks.run(pytestprog( 0.0, "plus1", file_in="input", file_out="output"), outputs=["output"], inputs=[(b1, "input")]) loom_env.start(1) result = loom_env.submit_and_gather(c1) assert result == b"cdef" * 100
def test_run_hostname(loom_env): loom_env.start(1) TASKS_COUNT = 100 ts = [tasks.run("hostname") for i in range(TASKS_COUNT)] array = tasks.array_make(ts) loom_env.submit_and_gather(array)
def test_py_task_deserialization(loom_env): @tasks.py_task() def f(): return b"" loom_env.start(2) ts = [f(), tasks.run("ls"), f(), f(), f()] result = loom_env.submit_and_gather(ts) assert len(result) == 5
def test_recover1(loom_env): loom_env.start(1) for i in range(10): a = tasks.run("ls /non-existing-dictionary") with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(a) b = tasks.const("XYZ") c = tasks.const("12345") d = tasks.merge((b, c)) assert loom_env.submit_and_gather(d) == b"XYZ12345"
def test_run_double_lines(loom_env): COUNT = 20000 a1 = tasks.const("afi" * 20000) b1 = tasks.run(pytestprog(0.0, "plus1"), stdin=a1) c1 = tasks.run(pytestprog(0.0, "plus1"), stdin=b1) a2 = tasks.const("kkllmm" * 20000) b2 = tasks.run(pytestprog(0.0, "plus1"), stdin=a2) c2 = tasks.run(pytestprog(0.0), stdin=b2) result = tasks.merge((c1, c2, a2)) expect = b"chk" * COUNT + b"llmmnn" * COUNT + b"kkllmm" * COUNT for i in range(1, 4): # print "Runnig for {}".format(i) cleanup() loom_env.start(i) r = loom_env.submit_and_gather(result) assert r == expect
def test_reconnect2(loom_env): loom_env.start(1) a = tasks.run("sleep 1") loom_env.client.submit_one(a) loom_env.close_client() a = tasks.const("abc") b = tasks.const("xyz") c = tasks.const("123") d = tasks.merge((a, b, c)) result = loom_env.submit_and_gather(d) assert result == b"abcxyz123" loom_env.check_final_state()
def test_cv_iris(loom_env): CHUNKS = 15 CHUNK_SIZE = 150 // CHUNKS # There are 150 irises loom_env.start(4, 4) loom_env.info = False data = tasks.open(IRIS_DATA) data = tasks.run(("sort", "--random-sort", "-"), [(data, None)]) lines = tasks.split(data) chunks = [tasks.slice(lines, i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE) for i in range(CHUNKS)] trainsets = [tasks.merge(chunks[:i] + chunks[i + 1:]) for i in range(CHUNKS)] models = [] for i, ts in enumerate(trainsets): model = tasks.run("svm-train data", [(ts, "data")], ["data.model"]) model.label = "svm-train: {}".format(i) models.append(model) predict = [] for chunk, model in zip(chunks, models): task = tasks.run("svm-predict testdata model out", [(chunk, "testdata"), (model, "model")]) task.label = "svm-predict" predict.append(task) loom_env.set_trace("mytrace") results = loom_env.submit_and_gather(predict) assert len(results) == CHUNKS for line in results: assert line.startswith(b"Accuracy = ")
def test_run_separated_4cpu_tasks_4_cpu(loom_env): loom_env.start(1, cpus=4) args = pytestprog(0.3, stamp=True) ts = [tasks.run(args, request=tasks.cpus(4)) for i in range(4)] results = loom_env.submit_and_gather(ts) starts = [] for result in results: line1, line2 = result.strip().split(b"\n") starts.append(str2datetime(line1)) for i in range(len(starts)): for j in range(len(starts)): if i == j: continue a = starts[i] b = starts[j] c = abs(a - b) assert 0.3 < c.total_seconds() < 1.40
def test_output_failed(loom_env): loom_env.start(1) a = tasks.run("ls /", outputs=["xxx"]) with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(a)
def test_program_failed(loom_env): loom_env.start(1) a = tasks.run("ls /non-existing-dictionary") with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(a)
def f(a): return tasks.run("cat X", [(a, "X")])
def test_invalid_program(loom_env): loom_env.start(1) a = tasks.run("/usr/bin/non-existing-program") with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(a)
def f(a, b): return tasks.run("/bin/ls $X", [(b, "$X")])