def test_py_value(loom_env): @tasks.py_task() def to_str(a): return str(a.unwrap()) @tasks.py_task(context=True) def to_tuple(ctx, a): b = tuple(a.unwrap()) return ctx.wrap(b) @tasks.py_task() def join(a, b): return [a, b] loom_env.start(2) v = tasks.py_value([(1, 2)]) v2 = tasks.py_value(("30", None)) t = to_str(v) result = loom_env.submit_and_gather(t) assert result == b"[(1, 2)]" t = to_tuple(v) result = loom_env.submit_and_gather(t) assert result == ((1, 2), ) v.resource_request = cpu1 v2.resource_request = cpu1 t = join(v, v2) result = loom_env.submit_and_gather(t) assert result == [[(1, 2)], ("30", None)]
def test_data_save(loom_env): loom_env.start(1) a1 = tasks.const("ABC" * 10000) a2 = tasks.const(b"") a3 = tasks.run("ls") with pytest.raises(client.TaskFailed): b = tasks.save(a1, "test-file") loom_env.submit_and_gather(b) with pytest.raises(client.TaskFailed): b = tasks.save(a1, "/") loom_env.submit_and_gather(b) b1 = tasks.save(a1, loom_env.get_filename("file1")) b2 = tasks.save(a2, loom_env.get_filename("file2")) b3 = tasks.save(a3, loom_env.get_filename("file3")) result = loom_env.submit_and_gather((b1, b2, b3)) assert result == [b"", b"", b""] with open(loom_env.get_filename("file1")) as f: assert f.read() == "ABC" * 10000 with open(loom_env.get_filename("file2")) as f: assert f.read() == "" with open(loom_env.get_filename("file3")) as f: content = f.read().split("\n") assert "+err" in content assert "+out" in content
def recover2(loom_env, n): @tasks.py_task() def slow_task1(): import time time.sleep(0.4) raise Exception("FAIL_2") @tasks.py_task() def slow_task2(): import time time.sleep(0.4) return b"ABC" @tasks.py_task() def fail_task(): import time time.sleep(0.10) raise Exception("FAIL") loom_env.start(n) x1 = slow_task1() x2 = slow_task2() x3 = fail_task() x4 = tasks.merge((x1, x2, x3)) b = tasks.const("XYZ") c = tasks.const("12345") d = tasks.merge((b, c)) for i in range(6): with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(x4) assert loom_env.submit_and_gather(d) == b"XYZ12345"
def test_run_hostname(loom_env): loom_env.start(1) TASKS_COUNT = 100 ts = [tasks.run("hostname") for i in range(TASKS_COUNT)] array = tasks.array_make(ts) loom_env.submit_and_gather(array)
def test_get_failed(loom_env): loom_env.start(1) a = tasks.const("A") b = tasks.const("BB") c = tasks.const("CCC") array = tasks.array_make((a, b, c)) x = tasks.get(array, 3) with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(x)
def test_recover1(loom_env): loom_env.start(1) for i in range(10): a = tasks.run("ls /non-existing-dictionary") with pytest.raises(client.TaskFailed): loom_env.submit_and_gather(a) b = tasks.const("XYZ") c = tasks.const("12345") d = tasks.merge((b, c)) assert loom_env.submit_and_gather(d) == b"XYZ12345"
def test_py_fail_invalid_result(loom_env): def f(): return 42.0 loom_env.start(1) a = tasks.py_call(tasks.py_value(f), ()) with pytest.raises(TaskFailed): loom_env.submit_and_gather(a)
def test_py_fail_too_few_args(loom_env): def f(a): return "ABC" loom_env.start(1) a = tasks.py_call(tasks.py_value(f), ()) with pytest.raises(TaskFailed): loom_env.submit_and_gather(a)
def test_py_fail_too_many_args(loom_env): def g(): return "Test" loom_env.start(1) c = tasks.const("ABC") a = tasks.py_call(tasks.py_value(g), (c, )) with pytest.raises(TaskFailed): loom_env.submit_and_gather(a)
def test_reconnect(loom_env): loom_env.start(2) code = """ import sys sys.path.insert(0, "{LOOM_PYTHON}") from loom.client import Client, tasks client = Client("localhost", {PORT}) a = tasks.run("sleep 1") b = tasks.run("sleep 1") c = tasks.run("sleep 1") d = tasks.run("sleep 1") fs = client.submit((a, b, c, d)) print(client.gather(fs)) """.format(LOOM_PYTHON=LOOM_PYTHON, PORT=loom_env.PORT) p = loom_env.independent_python(code) time.sleep(0.6) assert not p.poll() p.kill() p = loom_env.independent_python(code) time.sleep(0.6) assert not p.poll() p.kill() p = loom_env.independent_python(code) time.sleep(0.2) assert not p.poll() p.kill() time.sleep(0.2) a = tasks.run("ls") result = loom_env.submit_and_gather(a) assert result loom_env.check_final_state() a = tasks.const("abc") b = tasks.const("xyz") c = tasks.const("123") d = tasks.merge((a, b, c)) result = loom_env.submit_and_gather(d) assert result == b"abcxyz123" a = tasks.run("ls") result = loom_env.submit_and_gather(a) assert result loom_env.check_final_state()
def test_run_variable2(loom_env): a = tasks.const("123") b = tasks.const("456") c = tasks.run("/bin/echo $xyz xyz $ab $c", inputs=[(a, "$xyz"), (b, "$c")]) loom_env.start(1) result = loom_env.submit_and_gather(c) assert result == b"123 xyz $ab 456\n"
def test_single_result(loom_env): loom_env.start(1) a = tasks.const("ABCDE") b = tasks.const("123") c = tasks.merge((a, b)) assert b"ABCDE123" == loom_env.submit_and_gather(c) loom_env.check_final_state()
def test_py_array(loom_env): loom_env.start(1) @tasks.py_task() def t1(array): # len assert len(array) == 3 # indexing s = array[1].read() # iteration for x in array: s += x.read() return s a = tasks.const("A") b = tasks.const("BBBB") c = tasks.const("CCCCCCC") array = tasks.array_make((a, b, c)) x = t1(array) result = loom_env.submit_and_gather(x) assert result == b"BBBBABBBBCCCCCCC"
def test_checkpoint_load(loom_env): loom_env.start(1) path1 = os.path.join(LOOM_TEST_BUILD_DIR, "f1.txt") path2 = os.path.join(LOOM_TEST_BUILD_DIR, "f2.txt") path3 = os.path.join(LOOM_TEST_BUILD_DIR, "f3.txt") path4 = os.path.join(LOOM_TEST_BUILD_DIR, "f4.txt") path5 = os.path.join(LOOM_TEST_BUILD_DIR, "nonexisting") for i, p in enumerate((path1, path2, path3, path4)): with open(p, "w") as f: f.write("[{}]".format(i + 1)) t1 = tasks.const("$t1$") t1.checkpoint_path = path1 # This shoud load: [1] t2 = tasks.const("$t2$") t2.checkpoint_path = path2 # This shoud load: [2] t3 = tasks.const("$t3$") t4 = tasks.const("$t4$") x1 = tasks.merge((t1, t2, t3)) # [1][2]$t3$ x2 = tasks.merge((t1, x1)) x2.checkpoint_path = path3 # loaded as [3] x3 = tasks.merge((t4, t4)) x3.checkpoint_path = path4 # loaded as [4] x4 = tasks.merge((x3, x1, x2, t1, t2, t3)) x4.checkpoint_path = path5 assert loom_env.submit_and_gather( x4, load=True) == b'[4][1][2]$t3$[3][1][2]$t3$'
def test_py_direct_args(loom_env): @tasks.py_task(n_direct_args=1) def t1(p, a): return p + a.read() @tasks.py_task(n_direct_args=3) def t2(*args): return str(len(args)) @tasks.py_task(n_direct_args=1) def t3(x, y): return x * y.read() loom_env.start(1) c = tasks.const("ABC") a1 = t1(b"123", c) a2 = t2("123", "222", "1231231") a3 = t2({"x": 10}, None, True, c, c, c, c) a4 = t3(3, c) r1, r2, r3, r4 = loom_env.submit_and_gather((a1, a2, a3, a4)) assert r1 == b"123ABC" assert r2 == b"3" assert r3 == b"7" assert r4 == b"ABCABCABC"
def test_more_same_results(loom_env): loom_env.start(1) a = tasks.const("ABCDE") b = tasks.const("123") c = tasks.merge((a, b)) assert [b"ABCDE123", b"ABCDE123"] == loom_env.submit_and_gather([c, c]) loom_env.check_final_state()
def test_fail_and_report(loom_env): @tasks.py_task() def sleep(): import time time.sleep(1) return b"" @tasks.py_task() def fail(a): import time time.sleep(0.1) raise Exception("FAIL") loom_env.start(4) a = tasks.const("ABC") with pytest.raises(client.TaskFailed): loom_env.submit_and_gather((sleep(), sleep(), sleep(), fail(a)))
def test_crash_clean_worker(loom_env): loom_env.start(2) loom_env.kill_worker(0) a = tasks.const("ABCDE") b = tasks.const("123") c = tasks.merge((a, b)) assert b"ABCDE123" == loom_env.submit_and_gather(c)
def test_py_task_deserialization(loom_env): @tasks.py_task() def f(): return b"" loom_env.start(2) ts = [f(), tasks.run("ls"), f(), f(), f()] result = loom_env.submit_and_gather(ts) assert len(result) == 5
def test_py_wrap_wrapped(loom_env): @tasks.py_task(context=True) def task(ctx, a): return ctx.wrap(a) loom_env.start(2) v = tasks.py_value("ABC") t = task(v) result = loom_env.submit_and_gather(t) assert result == "ABC"
def test_py_redirect3(loom_env): def f(a): return tasks.run("cat X", [(a, "X")]) loom_env.start(1) c = tasks.const("DataData") a = tasks.py_call(tasks.py_value(f), (c, )) result = loom_env.submit_and_gather(a) assert b"DataData" in result
def test_open_and_merge(loom_env): a = tasks.open(FILE1) b = tasks.open(FILE2) c = tasks.merge((a, b)) loom_env.start(1) result = loom_env.submit_and_gather(c) expect = bytes( "This is file 1\n" + "\n".join("Line {}".format(i) for i in range(1, 13)) + "\n", "ascii") assert result == expect
def test_py_redirect1(loom_env): def f(a, b): return tasks.merge((a, b)) loom_env.start(1) c = tasks.const("ABC") d = tasks.const("12345") a = tasks.py_call(tasks.py_value(f), (c, d)) result = loom_env.submit_and_gather(a) assert result == b"ABC12345"
def test_checkpoint_basic(loom_env): loom_env.start(1) t1 = tasks.const("abcd") t2 = tasks.const("XYZ") t3 = tasks.merge((t1, t2)) path = os.path.join(LOOM_TEST_BUILD_DIR, "test.data") t3.checkpoint_path = path assert b"abcdXYZ" == loom_env.submit_and_gather(t3) with open(path, "rb") as f: assert f.read() == b"abcdXYZ" assert not os.path.isfile(path + ".loom.tmp")
def test_py_redirect2(loom_env): def f(a, b): return tasks.run("/bin/ls $X", [(b, "$X")]) loom_env.start(1) c = tasks.const("abcdef") d = tasks.const("/") a = tasks.py_call(tasks.py_value(f), (c, d)) result = loom_env.submit_and_gather(a) assert b"bin\n" in result assert b"usr\n" in result
def test_index_transfer(loom_env): loom_env.start(2) a = tasks.const("AAA\nBBB\n\nC\nDDDDDDDDDDD\nEEE") b = tasks.const("E\n\n") a2 = tasks.split(a) b2 = tasks.split(b) d = tasks.array_make((a2, b2)) x = tasks.get(d, 0) r1 = tasks.get(x, 0) r2 = tasks.get(x, 1) assert loom_env.submit_and_gather((r1, r2)) == [b"AAA\n", b"BBB\n"]
def test_py_multiple_return(loom_env): loom_env.start(1) @tasks.py_task() def t1(a, b): return a, "x", b, "yyy", [b, b, "z"] a = tasks.const("A") b = tasks.const("BBBB") c = t1(a, b) result = loom_env.submit_and_gather(c) assert result == [b"A", b"x", b"BBBB", b"yyy", [b"BBBB", b"BBBB", b"z"]]
def test_future_reuse1(loom_env): loom_env.start(1) a = tasks.const("abc") b = tasks.const("xyz") c = tasks.merge((a, b)) f = loom_env.client.submit_one(c) d = tasks.const("ijk") g = tasks.merge((d, f)) result = loom_env.submit_and_gather(g) assert result == b"ijkabcxyz" assert f.gather() == b"abcxyz" loom_env.check_final_state()
def test_size_and_length(loom_env): loom_env.start(1) text = "12345" * 5 a1 = tasks.const(text) a2 = tasks.array_make((a1, a1)) b1 = tasks.size(a1) c1 = tasks.length(a1) b2 = tasks.size(a2) c2 = tasks.length(a2) u64 = struct.Struct("<Q") [25, 0, 50, 2] == map(lambda x: u64.unpack(x)[0], loom_env.submit_and_gather((b1, c1, b2, c2)))
def test_reconnect2(loom_env): loom_env.start(1) a = tasks.run("sleep 1") loom_env.client.submit_one(a) loom_env.close_client() a = tasks.const("abc") b = tasks.const("xyz") c = tasks.const("123") d = tasks.merge((a, b, c)) result = loom_env.submit_and_gather(d) assert result == b"abcxyz123" loom_env.check_final_state()