def test_execute_cpus(test_env):
test_env.start(1, n_cpus=2)
with test_env.client.new_session() as s:
tasks.execute("sleep 1", cpus=2)
tasks.execute("sleep 1", cpus=2)
s.submit()
test_env.assert_duration(1.9, 2.3, lambda: s.wait_all())
def test_execute_outputs(test_env):
"Specify program content type on spec and instantiation."
obj = ["1", 2.0, {'a': 42}]
test_env.start(1)
with test_env.client.new_session() as s:
# No content type
t1a = tasks.execute(
["cat", Input("somefile", dataobj=pickled(obj))], stdout=Output())
t1a.output.keep()
# Static content-type by instantiation
t1b = tasks.execute(
["cat", Input("somefile", dataobj=pickled(obj))],
stdout=Output(content_type='pickle'))
t1b.output.keep()
# Stdin specification
t1c = tasks.execute(["cat"],
stdin=Input("somefile", dataobj=pickled(obj)),
stdout=Output(content_type='pickle'))
t1c.output.keep()
# Auto input naming
t1d = tasks.execute(["cat", Input(dataobj=pickled(obj))],
stdout=Output(content_type='pickle'))
t1d.output.keep()
s.submit()
assert t1b.output.fetch().load() == obj
assert t1c.output.fetch().load() == obj
assert t1d.output.fetch().load() == obj
assert t1a.output.content_type is None
with pytest.raises(RainException):
t1a.output.fetch().load()
def test_execute_positional_input(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
t0 = tasks.execute("ls /", stdout=True)
t1 = tasks.execute(("split", "-d", "-n", "l/2", t0),
output_files=["x00", "x01"])
t1.outputs["x00"].keep()
t1.outputs["x01"].keep()
s.submit()
t1.outputs["x00"].fetch()
t1.outputs["x01"].fetch()
def test_execute_positional_output(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
t0 = tasks.execute("ls /", stdout=True)
t1 = tasks.execute(("tee", Output("file")), stdin=t0, stdout="stdout")
t1.outputs["file"].keep()
t1.outputs["stdout"].keep()
s.submit()
f = t1.outputs["file"].fetch()
s = t1.outputs["stdout"].fetch()
assert f.get_bytes() == s.get_bytes()
def test_execute_termination(test_env):
test_env.start(1)
import time
with test_env.client.new_session() as s:
tasks.execute("sleep 5")
s.submit()
time.sleep(0.5)
with test_env.client.new_session() as s:
t1 = tasks.concat((blob("a"), blob("b")))
t1.keep_outputs()
s.submit()
r = test_env.assert_max_duration(0.2, lambda: t1.output.fetch())
assert b"ab" == r.get_bytes()
def test_fetch_outputs(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
t0 = tasks.execute("ls /", stdout=True)
t1 = tasks.execute(("split", "-d", "-n", "l/2", t0),
output_paths=["x00", "x01"])
t2 = tasks.execute("ls /", stdout=True)
t2.keep_outputs()
t1.keep_outputs()
s.submit()
a = t2.output.fetch()
b = t1.fetch_outputs()
assert len(a.get_bytes()) > 4
assert b[0].get_bytes() + b[1].get_bytes() == a.get_bytes()
def test_program_empty_output(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
task = tasks.execute(["echo", "-n"], stdout=True)
task.output.keep()
s.submit()
assert task.output.fetch().get_bytes() == b""
def test_program_write_input(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
x = blob(b"abc")
t = tasks.execute("ls",
input_paths=[Input("x", dataobj=x)],
output_paths=["x"]).output
t.keep()
t2 = tasks.execute("echo 'xyz' > x",
input_paths=[Input("x", dataobj=t, write=True)],
output_paths=["x"],
shell=True).output
t2.keep()
s.submit()
assert t2.fetch().get_bytes() == b"xyz\n"
assert t.fetch().get_bytes() == b"abc"
def test_slice_directory2(test_env):
os.mkdir("toplevel")
with open("toplevel/file1.txt", "w") as f:
f.write("My data 1")
os.mkdir("toplevel/dir1")
os.mkdir("toplevel/dir1/dir2")
with open("toplevel/dir1/dir2/file2.txt", "w") as f:
f.write("My data 2")
test_env.start(1, delete_list_timeout=0)
with test_env.client.new_session() as s:
d = directory("toplevel")
# Force fs mapping
d = tasks.execute("ls",
input_paths=[InputDir("d", dataobj=d)],
output_paths=[OutputDir("d")])
a1 = tasks.slice_directory(d, "file1.txt")
a1.output.keep()
a2 = tasks.slice_directory(d, "dir1/")
a2.output.keep()
a3 = tasks.slice_directory(d, "dir1/")
a3.output.keep()
a4 = tasks.slice_directory(d, "dir1/dir2/file2.txt")
a4.output.keep()
s.submit()
assert b"My data 1" == a1.output.fetch().get_bytes()
a2.output.fetch().write("result2")
with open("result2/dir2/file2.txt") as f:
assert f.read() == "My data 2"
a3.output.fetch().write("result3")
with open("result2/dir2/file2.txt") as f:
assert f.read() == "My data 2"
assert b"My data 2" == a4.output.fetch().get_bytes()
def test_execute_sleep_1(test_env):
"""Sleep followed by wait"""
test_env.start(1)
with test_env.client.new_session() as s:
t1 = tasks.execute("sleep 1")
s.submit()
test_env.assert_duration(0.99, 1.1, lambda: t1.wait())
def test_execute_stdout_only(test_env):
"""Capturing stdout"""
test_env.start(1)
with test_env.client.new_session() as s:
t1 = tasks.execute("ls /", stdout="output")
t1.output.keep()
s.submit()
assert b"etc\n" in t1.output.fetch().get_bytes()
def test_execute_fail(test_env):
"""Setting input file for program"""
test_env.start(1)
args = ("ls", "/non-existing-dir")
with test_env.client.new_session() as s:
t1 = tasks.execute(args, stdout="output")
s.submit()
pytest.raises(RainException, lambda: t1.wait())
def test_program_link_readonly(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
x = blob(b"abc")
t = tasks.execute("ls",
input_paths=[Input("x", dataobj=x)],
output_paths=["x"])
t.output.keep()
s.submit()
s.wait_all()
tasks.execute("ls > x",
input_paths=[Input("x", dataobj=t.output)],
output_paths=["x"],
shell=True)
s.submit()
with pytest.raises(TaskException):
s.wait_all()
def test_execute_invalid_filename(test_env):
"""Setting input file for program"""
test_env.start(1)
args = ("/bin/non-existing-program", )
with test_env.client.new_session() as s:
t1 = tasks.execute(args, stdout="output")
t1.output.keep()
s.submit()
pytest.raises(RainException, lambda: t1.wait())
def test_execute_create_file(test_env):
"""Capturing file"""
test_env.start(1)
args = ("/bin/bash", "-c", "echo ABC > output.txt")
with test_env.client.new_session() as s:
t1 = tasks.execute(
args, output_files=[Output("my_output", path="output.txt")])
t1.outputs["my_output"].keep()
s.submit()
assert t1.outputs["my_output"].fetch().get_bytes() == b"ABC\n"
def test_cpp_hello_file(test_env):
test_env.start(1, executor="cpptester")
with test_env.client.new_session() as s:
d1 = blob("WORLD")
t0 = tasks.execute("ls",
input_paths=[Input("d1", dataobj=d1)],
output_paths=[Output("d1")])
t1 = cpp_hello(t0.output)
t1.output.keep()
s.submit()
assert t1.output.fetch().get_bytes() == b"Hello WORLD!"
def test_execute_input_file(test_env):
"""Setting input file for program"""
test_env.start(1)
with test_env.client.new_session() as s:
t1 = tasks.execute(
("/bin/grep", "ab", Input("in1",
dataobj=blob("abc\nNOTHING\nabab"))),
stdout="output")
t1.output.keep()
s.submit()
assert t1.output.fetch().get_bytes() == b"abc\nabab\n"
def test_execute_stdin(test_env):
"""Setting input file for program"""
test_env.start(1)
args = ("/bin/grep", "ab")
with test_env.client.new_session() as s:
t1 = tasks.execute(args,
stdin=blob("abc\nNOTHING\nabab"),
stdout="output")
t1.output.keep()
s.submit()
assert t1.output.fetch().get_bytes() == b"abc\nabab\n"
def test_execute_shell(test_env):
test_env.start(1)
p1 = Program(("echo", "$HOME"), stdout=True)
p2 = Program(("echo", "$HOME"), stdout=True, shell=True)
with test_env.client.new_session() as s:
t1 = tasks.execute(("echo", "$HOME"), stdout=True)
t1.output.keep()
t2 = tasks.execute(("echo", "$HOME"), stdout=True, shell=True)
t2.output.keep()
t3 = p1()
t3.output.keep()
t4 = p2()
t4.output.keep()
s.submit()
assert b"$HOME\n" == t1.output.fetch().get_bytes()
assert (os.getenv("HOME") +
"\n").encode() == t2.output.fetch().get_bytes()
assert b"$HOME\n" == t3.output.fetch().get_bytes()
assert (os.getenv("HOME") +
"\n").encode() == t4.output.fetch().get_bytes()
def test_make_directory(test_env):
test_env.start(1, delete_list_timeout=0)
# TODO: EMPTY DIR os.mkdir("empty")
os.mkdir("mydir3")
with open("mydir3/file.txt", "w") as f:
f.write("My data 4")
with test_env.client.new_session() as s:
b1 = blob(b"My data 1")
b2 = blob(b"My data 2")
b3 = blob(b"My data 3")
d1 = directory("mydir3")
# TODO: EMPTY DIR d2 = directory("empty")
t0 = tasks.execute(["/bin/cat", b1],
stdout=True,
input_paths=[InputDir("d1", dataobj=d1)],
output_paths=[OutputDir("d1")])
r = tasks.make_directory([
("myfile1", t0.outputs["stdout"]),
("mydir/mydir2/myfile2", b2),
("mydir/myfile3", b3),
("mydir/d1a", d1),
# ("mydir/d2", d2),
("mydir/d1b", t0.outputs["d1"]),
])
r.output.keep()
s.submit()
s.wait_all()
r.output.fetch().write("rdir")
with open(os.path.join(test_env.work_dir, "rdir", "myfile1")) as f:
assert f.read() == "My data 1"
with open(
os.path.join(test_env.work_dir, "rdir", "mydir", "mydir2",
"myfile2")) as f:
assert f.read() == "My data 2"
with open(os.path.join(test_env.work_dir, "rdir", "mydir",
"myfile3")) as f:
assert f.read() == "My data 3"
with open(os.path.join(test_env.work_dir, "rdir", "mydir",
"myfile3")) as f:
assert f.read() == "My data 3"
with open(
os.path.join(test_env.work_dir, "rdir", "mydir", "d1a",
"file.txt")) as f:
assert f.read() == "My data 4"
with open(
os.path.join(test_env.work_dir, "rdir", "mydir", "d1b",
"file.txt")) as f:
assert f.read() == "My data 4"
def test_python_datainstance_write(test_env):
@remote()
def remote_fn(ctx, input1, input2):
input1.write("test1")
input2.write("test2")
with open("test1") as f:
assert f.read() == "Data 1"
with open("test1", "w") as f:
f.write("New data 1")
with open("test1") as f:
assert f.read() == "New data 1"
with open("test2/file") as f:
assert f.read() == "Data 2"
with open("test2/file", "w") as f:
f.write("New data 2")
with open("test2/file") as f:
assert f.read() == "New data 2"
os.mkdir("test2/testdir")
os.unlink("test1")
os.unlink("test2/file")
return b""
test_env.start(1)
with test_env.client.new_session() as s:
d1 = blob(b"Data 1")
os.mkdir("dir")
with open("dir/file", "w") as f:
f.write("Data 2")
d2 = directory("dir")
remote_fn(d1, d2)
s.submit()
s.wait_all()
x = tasks.execute(
"ls",
input_paths=[Input("d1", dataobj=d1), InputDir("d2", dataobj=d2)],
output_paths=[Output("d1"), OutputDir("d2")])
remote_fn(x.outputs["d1"], x.outputs["d2"])
s.submit()
s.wait_all()
def test_task_export(test_env):
import os.path
test1 = os.path.join(test_env.work_dir, "TEST1")
test2 = os.path.join(test_env.work_dir, "TEST2")
test_env.start(1)
with test_env.client.new_session() as s:
a = blob("Hello ")
b = blob("World!")
tasks.export(tasks.concat((a, b)), test1)
tasks.export(tasks.execute("ls /", stdout="output"), test2)
s.submit()
s.wait_all()
with open(test1) as f:
assert f.read() == "Hello World!"
with open(test2) as f:
assert "bin" in f.read()
def test_dir_big(test_env):
data = b"01234567890" * 1024 * 1024
os.mkdir("dir")
with open("dir/file1", "wb") as f:
f.write(data)
test_env.start(1)
with test_env.client.new_session() as s:
d = directory("dir")
t = tasks.execute("cat d/file1",
input_paths=[InputDir("d", dataobj=d)],
stdout=True,
output_paths=[OutputDir("d")])
t.keep_outputs()
s.submit()
assert t.outputs["stdout"].fetch().get_bytes() == data
t.outputs["d"].fetch().write("result")
with open("result/file1", "rb") as f:
assert f.read() == data
def test_execute_with_dir(test_env):
path = os.path.join(test_env.work_dir, "test-dir")
os.mkdir(path)
os.mkdir(os.path.join(path, "a"))
os.mkdir(os.path.join(path, "a", "b"))
with open(os.path.join(path, "f.txt"), "w") as f:
f.write("Hello 1")
with open(os.path.join(path, "a", "b", "g.txt"), "w") as f:
f.write("Hello 2")
test_env.start(1)
with test_env.client.new_session() as s:
data = directory(path=path)
e = tasks.execute("find ./mydir",
input_paths=[InputDir("mydir", dataobj=data)],
output_paths=[
Output("f", path="mydir/f.txt"),
OutputDir("a", path="mydir/a")
],
stdout=True)
e.keep_outputs()
s.submit()
s.wait_all()
result = set(
e.outputs["stdout"].fetch().get_bytes().strip().split(b"\n"))
assert b"./mydir" in result
assert b"./mydir/a" in result
assert b"./mydir/a/b" in result
assert b"./mydir/f.txt" in result
assert b"./mydir/a/b/g.txt" in result
assert e.outputs["f"].fetch().get_bytes() == b"Hello 1"
e.outputs["a"].fetch().write(os.path.join(test_env.work_dir, "result"))
assert os.path.isfile(
os.path.join(test_env.work_dir, "result", "b", "g.txt"))
