def test_single_result(loom_env):
loom_env.start(1)
a = tasks.const("ABCDE")
b = tasks.const("123")
c = tasks.merge((a, b))
assert b"ABCDE123" == loom_env.submit_and_gather(c)
loom_env.check_final_state()
def test_py_direct_args(loom_env):
@tasks.py_task(n_direct_args=1)
def t1(p, a):
return p + a.read()
@tasks.py_task(n_direct_args=3)
def t2(*args):
return str(len(args))
@tasks.py_task(n_direct_args=1)
def t3(x, y):
return x * y.read()
loom_env.start(1)
c = tasks.const("ABC")
a1 = t1(b"123", c)
a2 = t2("123", "222", "1231231")
a3 = t2({"x": 10}, None, True, c, c, c, c)
a4 = t3(3, c)
r1, r2, r3, r4 = loom_env.submit_and_gather((a1, a2, a3, a4))
assert r1 == b"123ABC"
assert r2 == b"3"
assert r3 == b"7"
assert r4 == b"ABCABCABC"
def test_py_value(loom_env):
@tasks.py_task()
def to_str(a):
return str(a.unwrap())
@tasks.py_task(context=True)
def to_tuple(ctx, a):
b = tuple(a.unwrap())
return ctx.wrap(b)
@tasks.py_task()
def join(a, b):
return [a, b]
loom_env.start(2)
v = tasks.py_value([(1, 2)])
v2 = tasks.py_value(("30", None))
t = to_str(v)
result = loom_env.submit_and_gather(t)
assert result == b"[(1, 2)]"
t = to_tuple(v)
result = loom_env.submit_and_gather(t)
assert result == ((1, 2), )
v.resource_request = cpu1
v2.resource_request = cpu1
t = join(v, v2)
result = loom_env.submit_and_gather(t)
assert result == [[(1, 2)], ("30", None)]
def test_rewrap_test(loom_env):
@tasks.py_task(context=True, n_direct_args=4)
def init(ctx):
content = [1, 2, 3]
return ctx.wrap(content)
@tasks.py_task(context=True)
def center(ctx, train_test):
train, test = [t.unwrap() for t in train_test]
return [ctx.wrap(t) for t in (train, test)]
@tasks.py_task(context=True)
def remove_empty_rows(ctx, train, test):
train = list(train.unwrap())
test = list(test.unwrap())
return [ctx.wrap(t) for t in (train, test)]
train = init()
test = init()
mean_task = center(remove_empty_rows(train, test))
smurff_tasks = [mean_task]
loom_env.start(1)
futures = loom_env.client.submit(smurff_tasks)
results = loom_env.client.gather(futures)
def test_checkpoint_load(loom_env):
loom_env.start(1)
path1 = os.path.join(LOOM_TEST_BUILD_DIR, "f1.txt")
path2 = os.path.join(LOOM_TEST_BUILD_DIR, "f2.txt")
path3 = os.path.join(LOOM_TEST_BUILD_DIR, "f3.txt")
path4 = os.path.join(LOOM_TEST_BUILD_DIR, "f4.txt")
path5 = os.path.join(LOOM_TEST_BUILD_DIR, "nonexisting")
for i, p in enumerate((path1, path2, path3, path4)):
with open(p, "w") as f:
f.write("[{}]".format(i + 1))
t1 = tasks.const("$t1$")
t1.checkpoint_path = path1 # This shoud load: [1]
t2 = tasks.const("$t2$")
t2.checkpoint_path = path2 # This shoud load: [2]
t3 = tasks.const("$t3$")
t4 = tasks.const("$t4$")
x1 = tasks.merge((t1, t2, t3)) # [1][2]$t3$
x2 = tasks.merge((t1, x1))
x2.checkpoint_path = path3 # loaded as [3]
x3 = tasks.merge((t4, t4))
x3.checkpoint_path = path4 # loaded as [4]
x4 = tasks.merge((x3, x1, x2, t1, t2, t3))
x4.checkpoint_path = path5
assert loom_env.submit_and_gather(
x4, load=True) == b'[4][1][2]$t3$[3][1][2]$t3$'
def test_py_array(loom_env):
loom_env.start(1)
@tasks.py_task()
def t1(array):
# len
assert len(array) == 3
# indexing
s = array[1].read()
# iteration
for x in array:
s += x.read()
return s
a = tasks.const("A")
b = tasks.const("BBBB")
c = tasks.const("CCCCCCC")
array = tasks.array_make((a, b, c))
x = t1(array)
result = loom_env.submit_and_gather(x)
assert result == b"BBBBABBBBCCCCCCC"
def recover2(loom_env, n):
@tasks.py_task()
def slow_task1():
import time
time.sleep(0.4)
raise Exception("FAIL_2")
@tasks.py_task()
def slow_task2():
import time
time.sleep(0.4)
return b"ABC"
@tasks.py_task()
def fail_task():
import time
time.sleep(0.10)
raise Exception("FAIL")
loom_env.start(n)
x1 = slow_task1()
x2 = slow_task2()
x3 = fail_task()
x4 = tasks.merge((x1, x2, x3))
b = tasks.const("XYZ")
c = tasks.const("12345")
d = tasks.merge((b, c))
for i in range(6):
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(x4)
assert loom_env.submit_and_gather(d) == b"XYZ12345"
def test_data_save(loom_env):
loom_env.start(1)
a1 = tasks.const("ABC" * 10000)
a2 = tasks.const(b"")
a3 = tasks.run("ls")
with pytest.raises(client.TaskFailed):
b = tasks.save(a1, "test-file")
loom_env.submit_and_gather(b)
with pytest.raises(client.TaskFailed):
b = tasks.save(a1, "/")
loom_env.submit_and_gather(b)
b1 = tasks.save(a1, loom_env.get_filename("file1"))
b2 = tasks.save(a2, loom_env.get_filename("file2"))
b3 = tasks.save(a3, loom_env.get_filename("file3"))
result = loom_env.submit_and_gather((b1, b2, b3))
assert result == [b"", b"", b""]
with open(loom_env.get_filename("file1")) as f:
assert f.read() == "ABC" * 10000
with open(loom_env.get_filename("file2")) as f:
assert f.read() == ""
with open(loom_env.get_filename("file3")) as f:
content = f.read().split("\n")
assert "+err" in content
assert "+out" in content
def test_future_fetch(loom_env):
loom_env.start(1)
a = tasks.const("ABCDE")
f = loom_env.client.submit_one(a)
assert f.fetch() == b"ABCDE"
f.release()
loom_env.check_final_state()
def test_run_variable2(loom_env):
a = tasks.const("123")
b = tasks.const("456")
c = tasks.run("/bin/echo $xyz xyz $ab $c", inputs=[(a, "$xyz"), (b, "$c")])
loom_env.start(1)
result = loom_env.submit_and_gather(c)
assert result == b"123 xyz $ab 456\n"
def test_more_same_results(loom_env):
loom_env.start(1)
a = tasks.const("ABCDE")
b = tasks.const("123")
c = tasks.merge((a, b))
assert [b"ABCDE123", b"ABCDE123"] == loom_env.submit_and_gather([c, c])
loom_env.check_final_state()
def test_future_wait(loom_env):
loom_env.start(1)
a = tasks.const("abcde")
f = loom_env.client.submit_one(a)
f.wait()
assert f.finished()
f.release()
loom_env.check_final_state()
def test_crash_clean_worker(loom_env):
loom_env.start(2)
loom_env.kill_worker(0)
a = tasks.const("ABCDE")
b = tasks.const("123")
c = tasks.merge((a, b))
assert b"ABCDE123" == loom_env.submit_and_gather(c)
def test_py_task_deserialization(loom_env):
@tasks.py_task()
def f():
return b""
loom_env.start(2)
ts = [f(), tasks.run("ls"), f(), f(), f()]
result = loom_env.submit_and_gather(ts)
assert len(result) == 5
def test_run_hostname(loom_env):
loom_env.start(1)
TASKS_COUNT = 100
ts = [tasks.run("hostname") for i in range(TASKS_COUNT)]
array = tasks.array_make(ts)
loom_env.submit_and_gather(array)
def test_get_failed(loom_env):
loom_env.start(1)
a = tasks.const("A")
b = tasks.const("BB")
c = tasks.const("CCC")
array = tasks.array_make((a, b, c))
x = tasks.get(array, 3)
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(x)
def test_py_fail_too_many_args(loom_env):
def g():
return "Test"
loom_env.start(1)
c = tasks.const("ABC")
a = tasks.py_call(tasks.py_value(g), (c, ))
with pytest.raises(TaskFailed):
loom_env.submit_and_gather(a)
def test_open_and_merge(loom_env):
a = tasks.open(FILE1)
b = tasks.open(FILE2)
c = tasks.merge((a, b))
loom_env.start(1)
result = loom_env.submit_and_gather(c)
expect = bytes(
"This is file 1\n" + "\n".join("Line {}".format(i)
for i in range(1, 13)) + "\n", "ascii")
assert result == expect
def test_py_fail_too_few_args(loom_env):
def f(a):
return "ABC"
loom_env.start(1)
a = tasks.py_call(tasks.py_value(f), ())
with pytest.raises(TaskFailed):
loom_env.submit_and_gather(a)
def test_py_fail_invalid_result(loom_env):
def f():
return 42.0
loom_env.start(1)
a = tasks.py_call(tasks.py_value(f), ())
with pytest.raises(TaskFailed):
loom_env.submit_and_gather(a)
def test_recover1(loom_env):
loom_env.start(1)
for i in range(10):
a = tasks.run("ls /non-existing-dictionary")
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(a)
b = tasks.const("XYZ")
c = tasks.const("12345")
d = tasks.merge((b, c))
assert loom_env.submit_and_gather(d) == b"XYZ12345"
def test_py_wrap_wrapped(loom_env):
@tasks.py_task(context=True)
def task(ctx, a):
return ctx.wrap(a)
loom_env.start(2)
v = tasks.py_value("ABC")
t = task(v)
result = loom_env.submit_and_gather(t)
assert result == "ABC"
def test_py_redirect3(loom_env):
def f(a):
return tasks.run("cat X", [(a, "X")])
loom_env.start(1)
c = tasks.const("DataData")
a = tasks.py_call(tasks.py_value(f), (c, ))
result = loom_env.submit_and_gather(a)
assert b"DataData" in result
def test_py_redirect1(loom_env):
def f(a, b):
return tasks.merge((a, b))
loom_env.start(1)
c = tasks.const("ABC")
d = tasks.const("12345")
a = tasks.py_call(tasks.py_value(f), (c, d))
result = loom_env.submit_and_gather(a)
assert result == b"ABC12345"
def test_checkpoint_basic(loom_env):
loom_env.start(1)
t1 = tasks.const("abcd")
t2 = tasks.const("XYZ")
t3 = tasks.merge((t1, t2))
path = os.path.join(LOOM_TEST_BUILD_DIR, "test.data")
t3.checkpoint_path = path
assert b"abcdXYZ" == loom_env.submit_and_gather(t3)
with open(path, "rb") as f:
assert f.read() == b"abcdXYZ"
assert not os.path.isfile(path + ".loom.tmp")
def test_py_redirect2(loom_env):
def f(a, b):
return tasks.run("/bin/ls $X", [(b, "$X")])
loom_env.start(1)
c = tasks.const("abcdef")
d = tasks.const("/")
a = tasks.py_call(tasks.py_value(f), (c, d))
result = loom_env.submit_and_gather(a)
assert b"bin\n" in result
assert b"usr\n" in result
def test_index_transfer(loom_env):
loom_env.start(2)
a = tasks.const("AAA\nBBB\n\nC\nDDDDDDDDDDD\nEEE")
b = tasks.const("E\n\n")
a2 = tasks.split(a)
b2 = tasks.split(b)
d = tasks.array_make((a2, b2))
x = tasks.get(d, 0)
r1 = tasks.get(x, 0)
r2 = tasks.get(x, 1)
assert loom_env.submit_and_gather((r1, r2)) == [b"AAA\n", b"BBB\n"]
def test_py_multiple_return(loom_env):
loom_env.start(1)
@tasks.py_task()
def t1(a, b):
return a, "x", b, "yyy", [b, b, "z"]
a = tasks.const("A")
b = tasks.const("BBBB")
c = t1(a, b)
result = loom_env.submit_and_gather(c)
assert result == [b"A", b"x", b"BBBB", b"yyy", [b"BBBB", b"BBBB", b"z"]]
def test_gather_all(loom_env):
loom_env.start(1)
a = tasks.const("abcde")
fa = loom_env.client.submit_one(a)
b = tasks.const("xyz")
fb = loom_env.client.submit_one(b)
assert [b"abcde", b"xyz"] == loom_env.client.gather((fa, fb))
assert fa.released()
assert fb.released()
fa.release()
fb.release()
loom_env.check_final_state()
def test_future_reuse1(loom_env):
loom_env.start(1)
a = tasks.const("abc")
b = tasks.const("xyz")
c = tasks.merge((a, b))
f = loom_env.client.submit_one(c)
d = tasks.const("ijk")
g = tasks.merge((d, f))
result = loom_env.submit_and_gather(g)
assert result == b"ijkabcxyz"
assert f.gather() == b"abcxyz"
loom_env.check_final_state()
