def test_execute_cpus(test_env): test_env.start(1, n_cpus=2) with test_env.client.new_session() as s: tasks.execute("sleep 1", cpus=2) tasks.execute("sleep 1", cpus=2) s.submit() test_env.assert_duration(1.9, 2.3, lambda: s.wait_all())
def test_execute_outputs(test_env): "Specify program content type on spec and instantiation." obj = ["1", 2.0, {'a': 42}] test_env.start(1) with test_env.client.new_session() as s: # No content type t1a = tasks.execute( ["cat", Input("somefile", dataobj=pickled(obj))], stdout=Output()) t1a.output.keep() # Static content-type by instantiation t1b = tasks.execute( ["cat", Input("somefile", dataobj=pickled(obj))], stdout=Output(content_type='pickle')) t1b.output.keep() # Stdin specification t1c = tasks.execute(["cat"], stdin=Input("somefile", dataobj=pickled(obj)), stdout=Output(content_type='pickle')) t1c.output.keep() # Auto input naming t1d = tasks.execute(["cat", Input(dataobj=pickled(obj))], stdout=Output(content_type='pickle')) t1d.output.keep() s.submit() assert t1b.output.fetch().load() == obj assert t1c.output.fetch().load() == obj assert t1d.output.fetch().load() == obj assert t1a.output.content_type is None with pytest.raises(RainException): t1a.output.fetch().load()
def test_execute_positional_input(test_env): test_env.start(1) with test_env.client.new_session() as s: t0 = tasks.execute("ls /", stdout=True) t1 = tasks.execute(("split", "-d", "-n", "l/2", t0), output_files=["x00", "x01"]) t1.outputs["x00"].keep() t1.outputs["x01"].keep() s.submit() t1.outputs["x00"].fetch() t1.outputs["x01"].fetch()
def test_execute_positional_output(test_env): test_env.start(1) with test_env.client.new_session() as s: t0 = tasks.execute("ls /", stdout=True) t1 = tasks.execute(("tee", Output("file")), stdin=t0, stdout="stdout") t1.outputs["file"].keep() t1.outputs["stdout"].keep() s.submit() f = t1.outputs["file"].fetch() s = t1.outputs["stdout"].fetch() assert f.get_bytes() == s.get_bytes()
def test_execute_termination(test_env): test_env.start(1) import time with test_env.client.new_session() as s: tasks.execute("sleep 5") s.submit() time.sleep(0.5) with test_env.client.new_session() as s: t1 = tasks.concat((blob("a"), blob("b"))) t1.keep_outputs() s.submit() r = test_env.assert_max_duration(0.2, lambda: t1.output.fetch()) assert b"ab" == r.get_bytes()
def test_fetch_outputs(test_env): test_env.start(1) with test_env.client.new_session() as s: t0 = tasks.execute("ls /", stdout=True) t1 = tasks.execute(("split", "-d", "-n", "l/2", t0), output_paths=["x00", "x01"]) t2 = tasks.execute("ls /", stdout=True) t2.keep_outputs() t1.keep_outputs() s.submit() a = t2.output.fetch() b = t1.fetch_outputs() assert len(a.get_bytes()) > 4 assert b[0].get_bytes() + b[1].get_bytes() == a.get_bytes()
def test_program_empty_output(test_env): test_env.start(1) with test_env.client.new_session() as s: task = tasks.execute(["echo", "-n"], stdout=True) task.output.keep() s.submit() assert task.output.fetch().get_bytes() == b""
def test_program_write_input(test_env): test_env.start(1) with test_env.client.new_session() as s: x = blob(b"abc") t = tasks.execute("ls", input_paths=[Input("x", dataobj=x)], output_paths=["x"]).output t.keep() t2 = tasks.execute("echo 'xyz' > x", input_paths=[Input("x", dataobj=t, write=True)], output_paths=["x"], shell=True).output t2.keep() s.submit() assert t2.fetch().get_bytes() == b"xyz\n" assert t.fetch().get_bytes() == b"abc"
def test_slice_directory2(test_env): os.mkdir("toplevel") with open("toplevel/file1.txt", "w") as f: f.write("My data 1") os.mkdir("toplevel/dir1") os.mkdir("toplevel/dir1/dir2") with open("toplevel/dir1/dir2/file2.txt", "w") as f: f.write("My data 2") test_env.start(1, delete_list_timeout=0) with test_env.client.new_session() as s: d = directory("toplevel") # Force fs mapping d = tasks.execute("ls", input_paths=[InputDir("d", dataobj=d)], output_paths=[OutputDir("d")]) a1 = tasks.slice_directory(d, "file1.txt") a1.output.keep() a2 = tasks.slice_directory(d, "dir1/") a2.output.keep() a3 = tasks.slice_directory(d, "dir1/") a3.output.keep() a4 = tasks.slice_directory(d, "dir1/dir2/file2.txt") a4.output.keep() s.submit() assert b"My data 1" == a1.output.fetch().get_bytes() a2.output.fetch().write("result2") with open("result2/dir2/file2.txt") as f: assert f.read() == "My data 2" a3.output.fetch().write("result3") with open("result2/dir2/file2.txt") as f: assert f.read() == "My data 2" assert b"My data 2" == a4.output.fetch().get_bytes()
def test_execute_sleep_1(test_env): """Sleep followed by wait""" test_env.start(1) with test_env.client.new_session() as s: t1 = tasks.execute("sleep 1") s.submit() test_env.assert_duration(0.99, 1.1, lambda: t1.wait())
def test_execute_stdout_only(test_env): """Capturing stdout""" test_env.start(1) with test_env.client.new_session() as s: t1 = tasks.execute("ls /", stdout="output") t1.output.keep() s.submit() assert b"etc\n" in t1.output.fetch().get_bytes()
def test_execute_fail(test_env): """Setting input file for program""" test_env.start(1) args = ("ls", "/non-existing-dir") with test_env.client.new_session() as s: t1 = tasks.execute(args, stdout="output") s.submit() pytest.raises(RainException, lambda: t1.wait())
def test_program_link_readonly(test_env): test_env.start(1) with test_env.client.new_session() as s: x = blob(b"abc") t = tasks.execute("ls", input_paths=[Input("x", dataobj=x)], output_paths=["x"]) t.output.keep() s.submit() s.wait_all() tasks.execute("ls > x", input_paths=[Input("x", dataobj=t.output)], output_paths=["x"], shell=True) s.submit() with pytest.raises(TaskException): s.wait_all()
def test_execute_invalid_filename(test_env): """Setting input file for program""" test_env.start(1) args = ("/bin/non-existing-program", ) with test_env.client.new_session() as s: t1 = tasks.execute(args, stdout="output") t1.output.keep() s.submit() pytest.raises(RainException, lambda: t1.wait())
def test_execute_create_file(test_env): """Capturing file""" test_env.start(1) args = ("/bin/bash", "-c", "echo ABC > output.txt") with test_env.client.new_session() as s: t1 = tasks.execute( args, output_files=[Output("my_output", path="output.txt")]) t1.outputs["my_output"].keep() s.submit() assert t1.outputs["my_output"].fetch().get_bytes() == b"ABC\n"
def test_cpp_hello_file(test_env): test_env.start(1, executor="cpptester") with test_env.client.new_session() as s: d1 = blob("WORLD") t0 = tasks.execute("ls", input_paths=[Input("d1", dataobj=d1)], output_paths=[Output("d1")]) t1 = cpp_hello(t0.output) t1.output.keep() s.submit() assert t1.output.fetch().get_bytes() == b"Hello WORLD!"
def test_execute_input_file(test_env): """Setting input file for program""" test_env.start(1) with test_env.client.new_session() as s: t1 = tasks.execute( ("/bin/grep", "ab", Input("in1", dataobj=blob("abc\nNOTHING\nabab"))), stdout="output") t1.output.keep() s.submit() assert t1.output.fetch().get_bytes() == b"abc\nabab\n"
def test_execute_stdin(test_env): """Setting input file for program""" test_env.start(1) args = ("/bin/grep", "ab") with test_env.client.new_session() as s: t1 = tasks.execute(args, stdin=blob("abc\nNOTHING\nabab"), stdout="output") t1.output.keep() s.submit() assert t1.output.fetch().get_bytes() == b"abc\nabab\n"
def test_execute_shell(test_env): test_env.start(1) p1 = Program(("echo", "$HOME"), stdout=True) p2 = Program(("echo", "$HOME"), stdout=True, shell=True) with test_env.client.new_session() as s: t1 = tasks.execute(("echo", "$HOME"), stdout=True) t1.output.keep() t2 = tasks.execute(("echo", "$HOME"), stdout=True, shell=True) t2.output.keep() t3 = p1() t3.output.keep() t4 = p2() t4.output.keep() s.submit() assert b"$HOME\n" == t1.output.fetch().get_bytes() assert (os.getenv("HOME") + "\n").encode() == t2.output.fetch().get_bytes() assert b"$HOME\n" == t3.output.fetch().get_bytes() assert (os.getenv("HOME") + "\n").encode() == t4.output.fetch().get_bytes()
def test_make_directory(test_env): test_env.start(1, delete_list_timeout=0) # TODO: EMPTY DIR os.mkdir("empty") os.mkdir("mydir3") with open("mydir3/file.txt", "w") as f: f.write("My data 4") with test_env.client.new_session() as s: b1 = blob(b"My data 1") b2 = blob(b"My data 2") b3 = blob(b"My data 3") d1 = directory("mydir3") # TODO: EMPTY DIR d2 = directory("empty") t0 = tasks.execute(["/bin/cat", b1], stdout=True, input_paths=[InputDir("d1", dataobj=d1)], output_paths=[OutputDir("d1")]) r = tasks.make_directory([ ("myfile1", t0.outputs["stdout"]), ("mydir/mydir2/myfile2", b2), ("mydir/myfile3", b3), ("mydir/d1a", d1), # ("mydir/d2", d2), ("mydir/d1b", t0.outputs["d1"]), ]) r.output.keep() s.submit() s.wait_all() r.output.fetch().write("rdir") with open(os.path.join(test_env.work_dir, "rdir", "myfile1")) as f: assert f.read() == "My data 1" with open( os.path.join(test_env.work_dir, "rdir", "mydir", "mydir2", "myfile2")) as f: assert f.read() == "My data 2" with open(os.path.join(test_env.work_dir, "rdir", "mydir", "myfile3")) as f: assert f.read() == "My data 3" with open(os.path.join(test_env.work_dir, "rdir", "mydir", "myfile3")) as f: assert f.read() == "My data 3" with open( os.path.join(test_env.work_dir, "rdir", "mydir", "d1a", "file.txt")) as f: assert f.read() == "My data 4" with open( os.path.join(test_env.work_dir, "rdir", "mydir", "d1b", "file.txt")) as f: assert f.read() == "My data 4"
def test_python_datainstance_write(test_env): @remote() def remote_fn(ctx, input1, input2): input1.write("test1") input2.write("test2") with open("test1") as f: assert f.read() == "Data 1" with open("test1", "w") as f: f.write("New data 1") with open("test1") as f: assert f.read() == "New data 1" with open("test2/file") as f: assert f.read() == "Data 2" with open("test2/file", "w") as f: f.write("New data 2") with open("test2/file") as f: assert f.read() == "New data 2" os.mkdir("test2/testdir") os.unlink("test1") os.unlink("test2/file") return b"" test_env.start(1) with test_env.client.new_session() as s: d1 = blob(b"Data 1") os.mkdir("dir") with open("dir/file", "w") as f: f.write("Data 2") d2 = directory("dir") remote_fn(d1, d2) s.submit() s.wait_all() x = tasks.execute( "ls", input_paths=[Input("d1", dataobj=d1), InputDir("d2", dataobj=d2)], output_paths=[Output("d1"), OutputDir("d2")]) remote_fn(x.outputs["d1"], x.outputs["d2"]) s.submit() s.wait_all()
def test_task_export(test_env): import os.path test1 = os.path.join(test_env.work_dir, "TEST1") test2 = os.path.join(test_env.work_dir, "TEST2") test_env.start(1) with test_env.client.new_session() as s: a = blob("Hello ") b = blob("World!") tasks.export(tasks.concat((a, b)), test1) tasks.export(tasks.execute("ls /", stdout="output"), test2) s.submit() s.wait_all() with open(test1) as f: assert f.read() == "Hello World!" with open(test2) as f: assert "bin" in f.read()
def test_dir_big(test_env): data = b"01234567890" * 1024 * 1024 os.mkdir("dir") with open("dir/file1", "wb") as f: f.write(data) test_env.start(1) with test_env.client.new_session() as s: d = directory("dir") t = tasks.execute("cat d/file1", input_paths=[InputDir("d", dataobj=d)], stdout=True, output_paths=[OutputDir("d")]) t.keep_outputs() s.submit() assert t.outputs["stdout"].fetch().get_bytes() == data t.outputs["d"].fetch().write("result") with open("result/file1", "rb") as f: assert f.read() == data
def test_execute_with_dir(test_env): path = os.path.join(test_env.work_dir, "test-dir") os.mkdir(path) os.mkdir(os.path.join(path, "a")) os.mkdir(os.path.join(path, "a", "b")) with open(os.path.join(path, "f.txt"), "w") as f: f.write("Hello 1") with open(os.path.join(path, "a", "b", "g.txt"), "w") as f: f.write("Hello 2") test_env.start(1) with test_env.client.new_session() as s: data = directory(path=path) e = tasks.execute("find ./mydir", input_paths=[InputDir("mydir", dataobj=data)], output_paths=[ Output("f", path="mydir/f.txt"), OutputDir("a", path="mydir/a") ], stdout=True) e.keep_outputs() s.submit() s.wait_all() result = set( e.outputs["stdout"].fetch().get_bytes().strip().split(b"\n")) assert b"./mydir" in result assert b"./mydir/a" in result assert b"./mydir/a/b" in result assert b"./mydir/f.txt" in result assert b"./mydir/a/b/g.txt" in result assert e.outputs["f"].fetch().get_bytes() == b"Hello 1" e.outputs["a"].fetch().write(os.path.join(test_env.work_dir, "result")) assert os.path.isfile( os.path.join(test_env.work_dir, "result", "b", "g.txt"))