def test_execute_outputs(test_env): "Specify program content type on spec and instantiation." obj = ["1", 2.0, {'a': 42}] test_env.start(1) with test_env.client.new_session() as s: # No content type t1a = tasks.execute( ["cat", Input("somefile", dataobj=pickled(obj))], stdout=Output()) t1a.output.keep() # Static content-type by instantiation t1b = tasks.execute( ["cat", Input("somefile", dataobj=pickled(obj))], stdout=Output(content_type='pickle')) t1b.output.keep() # Stdin specification t1c = tasks.execute(["cat"], stdin=Input("somefile", dataobj=pickled(obj)), stdout=Output(content_type='pickle')) t1c.output.keep() # Auto input naming t1d = tasks.execute(["cat", Input(dataobj=pickled(obj))], stdout=Output(content_type='pickle')) t1d.output.keep() s.submit() assert t1b.output.fetch().load() == obj assert t1c.output.fetch().load() == obj assert t1d.output.fetch().load() == obj assert t1a.output.content_type is None with pytest.raises(RainException): t1a.output.fetch().load()
def test_output_specs_num(test_env): @remote(outputs=3) def test1(ctx): return (b'HW', b'\xe1\xb9\xef\xeb', pickle.dumps([2.0, 3.0])) test_env.start(1) with test_env.client.new_session() as s: t1 = test1(outputs=[Output(), Output(content_type='text:latin2'), Output(content_type='pickle')]) t1.keep_outputs() s.submit() (a, b, c) = t1.fetch_outputs() assert b.load() == 'ášďë' assert c.load() == [2.0, 3.0]
def test_execute_create_file(test_env): """Capturing file""" test_env.start(1) args = ("/bin/bash", "-c", "echo ABC > output.txt") with test_env.client.new_session() as s: t1 = tasks.execute( args, output_files=[Output("my_output", path="output.txt")]) t1.outputs["my_output"].keep() s.submit() assert t1.outputs["my_output"].fetch().get_bytes() == b"ABC\n"
def test_output_detailed_specs(test_env): "Tests specifying content types for outputs and dynamic content types." obj = {1: 2, 3: [4, 5]} obj2 = [1.0, 2.0, True] @remote() def test1(ctx) -> (Output(encode='pickle', label='test_pickle', size_hint=0.1), Output(content_type='text:latin2'), "out_c", "out_d"): return (obj, b'\xe1\xb9\xef\xeb', pickle.dumps(obj2), ctx.blob(b"[42.0]", content_type='json')) @remote(outputs=[Output(encode='pickle', label='test_pickle', size_hint=0.1), Output(content_type='text:latin2'), "out_c", "out_d"]) def test2(ctx): return (obj, b'\xe1\xb9\xef\xeb', pickle.dumps(obj2), ctx.blob(b"[42.0]", content_type='json')) test_env.start(1) for test in (test1, test2): with test_env.client.new_session() as s: t = test(outputs=[None, None, Output(content_type='pickle'), "foo"]) t.keep_outputs() s.submit() (a, b, c, d) = t.fetch_outputs() print(t.outputs) assert t.outputs['foo'].label == "foo" assert a.load() == obj assert b.load() == 'ášďë' assert b.get_bytes() == b'\xe1\xb9\xef\xeb' assert c.load() == obj2 assert t.outputs['out_c'].fetch().load() == obj2 assert d.get_bytes() == b"[42.0]" # Dynamic content type assert d.content_type == 'json' assert d.load() == [42.0]
def test_cpp_hello_file(test_env): test_env.start(1, executor="cpptester") with test_env.client.new_session() as s: d1 = blob("WORLD") t0 = tasks.execute("ls", input_paths=[Input("d1", dataobj=d1)], output_paths=[Output("d1")]) t1 = cpp_hello(t0.output) t1.output.keep() s.submit() assert t1.output.fetch().get_bytes() == b"Hello WORLD!"
def test_hello_file(self, test_env): self.start(test_env) with test_env.client.new_session() as s: d1 = blob("WORLD") t0 = tasks.Execute("ls", input_paths=[Input("d1", dataobj=d1)], output_paths=[Output("d1")]) t1 = self.task_hello(t0.output) t1.output.keep() s.submit() assert t1.output.fetch().get_bytes() == b"Hello WORLD!"
def test_execute_positional_output(test_env): test_env.start(1) with test_env.client.new_session() as s: t0 = tasks.execute("ls /", stdout=True) t1 = tasks.execute(("tee", Output("file")), stdin=t0, stdout="stdout") t1.outputs["file"].keep() t1.outputs["stdout"].keep() s.submit() f = t1.outputs["file"].fetch() s = t1.outputs["stdout"].fetch() assert f.get_bytes() == s.get_bytes()
def test_program_outputs(test_env): "Specify program content type on spec and instantiation." obj = ["1", 2.0, {'a': 42}] program1 = Program(["cat", Input("i")], stdout="o") program2 = Program(["cat", Input("i", content_type='pickle')], stdout=Output(content_type='pickle')) test_env.start(1) with test_env.client.new_session() as s: # Dynamic content-type, forgotten by cat t1a = program1(i=pickled(obj)) t1a.output.keep() # Static content-type by instantiation t1b = program1(i=pickled(obj), output=Output(content_type='pickle')) t1b.output.keep() # No content type t1c = program1(i=blob(pickle.dumps(obj))) t1c.output.keep() # Static content-type by Program spec t2 = program2(i=pickled(obj)) t2.output.keep() s.submit() assert t1a.output.content_type is None with pytest.raises(RainException): assert t1a.output.fetch().load() == obj assert t1a.output.fetch().get_bytes() == pickle.dumps(obj) # TODO(gavento): Needs OutputSpec and Output merging # assert t1b.output.fetch().load() == obj assert t1c.output.content_type is None with pytest.raises(RainException): t1c.output.fetch().load() assert t1a.output.fetch().get_bytes() == pickle.dumps(obj) assert t2.output.fetch().load() == obj
def test_python_datainstance_write(test_env): @remote() def remote_fn(ctx, input1, input2): input1.write("test1") input2.write("test2") with open("test1") as f: assert f.read() == "Data 1" with open("test1", "w") as f: f.write("New data 1") with open("test1") as f: assert f.read() == "New data 1" with open("test2/file") as f: assert f.read() == "Data 2" with open("test2/file", "w") as f: f.write("New data 2") with open("test2/file") as f: assert f.read() == "New data 2" os.mkdir("test2/testdir") os.unlink("test1") os.unlink("test2/file") return b"" test_env.start(1) with test_env.client.new_session() as s: d1 = blob(b"Data 1") os.mkdir("dir") with open("dir/file", "w") as f: f.write("Data 2") d2 = directory("dir") remote_fn(d1, d2) s.submit() s.wait_all() x = tasks.Execute( "ls", input_paths=[Input("d1", dataobj=d1), InputDir("d2", dataobj=d2)], output_paths=[Output("d1"), OutputDir("d2")]) remote_fn(x.outputs["d1"], x.outputs["d2"]) s.submit() s.wait_all()
def test_execute_with_dir(test_env): path = os.path.join(test_env.work_dir, "test-dir") os.mkdir(path) os.mkdir(os.path.join(path, "a")) os.mkdir(os.path.join(path, "a", "b")) with open(os.path.join(path, "f.txt"), "w") as f: f.write("Hello 1") with open(os.path.join(path, "a", "b", "g.txt"), "w") as f: f.write("Hello 2") test_env.start(1) with test_env.client.new_session() as s: data = directory(path=path) e = tasks.Execute("find ./mydir", input_paths=[InputDir("mydir", dataobj=data)], output_paths=[ Output("f", path="mydir/f.txt"), OutputDir("a", path="mydir/a") ], stdout=True) e.keep_outputs() s.submit() s.wait_all() result = set( e.outputs["stdout"].fetch().get_bytes().strip().split(b"\n")) assert b"./mydir" in result assert b"./mydir/a" in result assert b"./mydir/a/b" in result assert b"./mydir/f.txt" in result assert b"./mydir/a/b/g.txt" in result assert e.outputs["f"].fetch().get_bytes() == b"Hello 1" e.outputs["a"].fetch().write(os.path.join(test_env.work_dir, "result")) assert os.path.isfile( os.path.join(test_env.work_dir, "result", "b", "g.txt"))
def test1( ctx ) -> (Output(encode='pickle', label='test_pickle', size_hint=0.1), Output(content_type='text-latin2'), "out_c", "out_d"): return (obj, b'\xe1\xb9\xef\xeb', pickle.dumps(obj2), ctx.blob(b"[42.0]", content_type='json'))