def test_future_reuse3(loom_env):
loom_env.start(3)
a = tasks.run("ls")
f = loom_env.client.submit_one(a)
COUNT = 1000
b = [tasks.run("ls") for i in range(COUNT)]
g = loom_env.client.submit(b)
result2 = f.fetch()
result = loom_env.submit_and_gather(tasks.merge(g))
loom_env.client.release(g)
f.release()
assert result2 * COUNT == result
loom_env.check_final_state()
def test_run_chain(loom_env):
const = b"ABC" * 10000000
a = tasks.const(const)
b1 = tasks.run(pytestprog(0.01, file_out="test"),
stdin=a, outputs=["test"])
b2 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"),
inputs=[(b1, "in")], outputs=["test"])
b3 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"),
inputs=[(b2, "in")], outputs=["test"])
b4 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"),
inputs=[(b3, "in")], outputs=["test"])
loom_env.start(1)
result = loom_env.submit_and_gather(b4)
assert result == const
def test_reconnect(loom_env):
loom_env.start(2)
code = """
import sys
sys.path.insert(0, "{LOOM_PYTHON}")
from loom.client import Client, tasks
client = Client("localhost", {PORT})
a = tasks.run("sleep 1")
b = tasks.run("sleep 1")
c = tasks.run("sleep 1")
d = tasks.run("sleep 1")
fs = client.submit((a, b, c, d))
print(client.gather(fs))
""".format(LOOM_PYTHON=LOOM_PYTHON, PORT=loom_env.PORT)
p = loom_env.independent_python(code)
time.sleep(0.6)
assert not p.poll()
p.kill()
p = loom_env.independent_python(code)
time.sleep(0.6)
assert not p.poll()
p.kill()
p = loom_env.independent_python(code)
time.sleep(0.2)
assert not p.poll()
p.kill()
time.sleep(0.2)
a = tasks.run("ls")
result = loom_env.submit_and_gather(a)
assert result
loom_env.check_final_state()
a = tasks.const("abc")
b = tasks.const("xyz")
c = tasks.const("123")
d = tasks.merge((a, b, c))
result = loom_env.submit_and_gather(d)
assert result == b"abcxyz123"
a = tasks.run("ls")
result = loom_env.submit_and_gather(a)
assert result
loom_env.check_final_state()
def test_data_save(loom_env):
loom_env.start(1)
a1 = tasks.const("ABC" * 10000)
a2 = tasks.const(b"")
a3 = tasks.run("ls")
with pytest.raises(client.TaskFailed):
b = tasks.save(a1, "test-file")
loom_env.submit_and_gather(b)
with pytest.raises(client.TaskFailed):
b = tasks.save(a1, "/")
loom_env.submit_and_gather(b)
b1 = tasks.save(a1, loom_env.get_filename("file1"))
b2 = tasks.save(a2, loom_env.get_filename("file2"))
b3 = tasks.save(a3, loom_env.get_filename("file3"))
result = loom_env.submit_and_gather((b1, b2, b3))
assert result == [b"", b"", b""]
with open(loom_env.get_filename("file1")) as f:
assert f.read() == "ABC" * 10000
with open(loom_env.get_filename("file2")) as f:
assert f.read() == ""
with open(loom_env.get_filename("file3")) as f:
content = f.read().split("\n")
assert "+err" in content
assert "+out" in content
def test_run_variable2(loom_env):
a = tasks.const("123")
b = tasks.const("456")
c = tasks.run("/bin/echo $xyz xyz $ab $c", inputs=[(a, "$xyz"), (b, "$c")])
loom_env.start(1)
result = loom_env.submit_and_gather(c)
assert result == b"123 xyz $ab 456\n"
def test_run_files(loom_env):
a1 = tasks.const("abcd" * 100)
b1 = tasks.run(
pytestprog(0.0, "plus1", file_out="myfile1"),
stdin=a1,
outputs=["myfile1"])
c1 = tasks.run(pytestprog(
0.0, "plus1", file_in="input", file_out="output"),
outputs=["output"],
inputs=[(b1, "input")])
loom_env.start(1)
result = loom_env.submit_and_gather(c1)
assert result == b"cdef" * 100
def test_run_hostname(loom_env):
loom_env.start(1)
TASKS_COUNT = 100
ts = [tasks.run("hostname") for i in range(TASKS_COUNT)]
array = tasks.array_make(ts)
loom_env.submit_and_gather(array)
def test_py_task_deserialization(loom_env):
@tasks.py_task()
def f():
return b""
loom_env.start(2)
ts = [f(), tasks.run("ls"), f(), f(), f()]
result = loom_env.submit_and_gather(ts)
assert len(result) == 5
def test_recover1(loom_env):
loom_env.start(1)
for i in range(10):
a = tasks.run("ls /non-existing-dictionary")
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(a)
b = tasks.const("XYZ")
c = tasks.const("12345")
d = tasks.merge((b, c))
assert loom_env.submit_and_gather(d) == b"XYZ12345"
def test_run_double_lines(loom_env):
COUNT = 20000
a1 = tasks.const("afi" * 20000)
b1 = tasks.run(pytestprog(0.0, "plus1"), stdin=a1)
c1 = tasks.run(pytestprog(0.0, "plus1"), stdin=b1)
a2 = tasks.const("kkllmm" * 20000)
b2 = tasks.run(pytestprog(0.0, "plus1"), stdin=a2)
c2 = tasks.run(pytestprog(0.0), stdin=b2)
result = tasks.merge((c1, c2, a2))
expect = b"chk" * COUNT + b"llmmnn" * COUNT + b"kkllmm" * COUNT
for i in range(1, 4):
# print "Runnig for {}".format(i)
cleanup()
loom_env.start(i)
r = loom_env.submit_and_gather(result)
assert r == expect
def test_reconnect2(loom_env):
loom_env.start(1)
a = tasks.run("sleep 1")
loom_env.client.submit_one(a)
loom_env.close_client()
a = tasks.const("abc")
b = tasks.const("xyz")
c = tasks.const("123")
d = tasks.merge((a, b, c))
result = loom_env.submit_and_gather(d)
assert result == b"abcxyz123"
loom_env.check_final_state()
def test_cv_iris(loom_env):
CHUNKS = 15
CHUNK_SIZE = 150 // CHUNKS # There are 150 irises
loom_env.start(4, 4)
loom_env.info = False
data = tasks.open(IRIS_DATA)
data = tasks.run(("sort", "--random-sort", "-"), [(data, None)])
lines = tasks.split(data)
chunks = [tasks.slice(lines, i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE)
for i in range(CHUNKS)]
trainsets = [tasks.merge(chunks[:i] + chunks[i + 1:])
for i in range(CHUNKS)]
models = []
for i, ts in enumerate(trainsets):
model = tasks.run("svm-train data",
[(ts, "data")], ["data.model"])
model.label = "svm-train: {}".format(i)
models.append(model)
predict = []
for chunk, model in zip(chunks, models):
task = tasks.run("svm-predict testdata model out",
[(chunk, "testdata"), (model, "model")])
task.label = "svm-predict"
predict.append(task)
loom_env.set_trace("mytrace")
results = loom_env.submit_and_gather(predict)
assert len(results) == CHUNKS
for line in results:
assert line.startswith(b"Accuracy = ")
def test_run_separated_4cpu_tasks_4_cpu(loom_env):
loom_env.start(1, cpus=4)
args = pytestprog(0.3, stamp=True)
ts = [tasks.run(args, request=tasks.cpus(4)) for i in range(4)]
results = loom_env.submit_and_gather(ts)
starts = []
for result in results:
line1, line2 = result.strip().split(b"\n")
starts.append(str2datetime(line1))
for i in range(len(starts)):
for j in range(len(starts)):
if i == j:
continue
a = starts[i]
b = starts[j]
c = abs(a - b)
assert 0.3 < c.total_seconds() < 1.40
def test_output_failed(loom_env):
loom_env.start(1)
a = tasks.run("ls /", outputs=["xxx"])
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(a)
def test_program_failed(loom_env):
loom_env.start(1)
a = tasks.run("ls /non-existing-dictionary")
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(a)
def f(a):
return tasks.run("cat X", [(a, "X")])
def test_invalid_program(loom_env):
loom_env.start(1)
a = tasks.run("/usr/bin/non-existing-program")
with pytest.raises(client.TaskFailed):
loom_env.submit_and_gather(a)
def f(a, b):
return tasks.run("/bin/ls $X", [(b, "$X")])