def test_execute_outputs(test_env):
"Specify program content type on spec and instantiation."
obj = ["1", 2.0, {'a': 42}]
test_env.start(1)
with test_env.client.new_session() as s:
# No content type
t1a = tasks.execute(
["cat", Input("somefile", dataobj=pickled(obj))], stdout=Output())
t1a.output.keep()
# Static content-type by instantiation
t1b = tasks.execute(
["cat", Input("somefile", dataobj=pickled(obj))],
stdout=Output(content_type='pickle'))
t1b.output.keep()
# Stdin specification
t1c = tasks.execute(["cat"],
stdin=Input("somefile", dataobj=pickled(obj)),
stdout=Output(content_type='pickle'))
t1c.output.keep()
# Auto input naming
t1d = tasks.execute(["cat", Input(dataobj=pickled(obj))],
stdout=Output(content_type='pickle'))
t1d.output.keep()
s.submit()
assert t1b.output.fetch().load() == obj
assert t1c.output.fetch().load() == obj
assert t1d.output.fetch().load() == obj
assert t1a.output.content_type is None
with pytest.raises(RainException):
t1a.output.fetch().load()
def test_output_specs_num(test_env):
@remote(outputs=3)
def test1(ctx):
return (b'HW', b'\xe1\xb9\xef\xeb', pickle.dumps([2.0, 3.0]))
test_env.start(1)
with test_env.client.new_session() as s:
t1 = test1(outputs=[Output(),
Output(content_type='text:latin2'),
Output(content_type='pickle')])
t1.keep_outputs()
s.submit()
(a, b, c) = t1.fetch_outputs()
assert b.load() == 'ášďë'
assert c.load() == [2.0, 3.0]
def test_execute_create_file(test_env):
"""Capturing file"""
test_env.start(1)
args = ("/bin/bash", "-c", "echo ABC > output.txt")
with test_env.client.new_session() as s:
t1 = tasks.execute(
args, output_files=[Output("my_output", path="output.txt")])
t1.outputs["my_output"].keep()
s.submit()
assert t1.outputs["my_output"].fetch().get_bytes() == b"ABC\n"
def test_output_detailed_specs(test_env):
"Tests specifying content types for outputs and dynamic content types."
obj = {1: 2, 3: [4, 5]}
obj2 = [1.0, 2.0, True]
@remote()
def test1(ctx) -> (Output(encode='pickle', label='test_pickle', size_hint=0.1),
Output(content_type='text:latin2'),
"out_c",
"out_d"):
return (obj, b'\xe1\xb9\xef\xeb', pickle.dumps(obj2),
ctx.blob(b"[42.0]", content_type='json'))
@remote(outputs=[Output(encode='pickle', label='test_pickle', size_hint=0.1),
Output(content_type='text:latin2'),
"out_c", "out_d"])
def test2(ctx):
return (obj, b'\xe1\xb9\xef\xeb', pickle.dumps(obj2),
ctx.blob(b"[42.0]", content_type='json'))
test_env.start(1)
for test in (test1, test2):
with test_env.client.new_session() as s:
t = test(outputs=[None,
None,
Output(content_type='pickle'),
"foo"])
t.keep_outputs()
s.submit()
(a, b, c, d) = t.fetch_outputs()
print(t.outputs)
assert t.outputs['foo'].label == "foo"
assert a.load() == obj
assert b.load() == 'ášďë'
assert b.get_bytes() == b'\xe1\xb9\xef\xeb'
assert c.load() == obj2
assert t.outputs['out_c'].fetch().load() == obj2
assert d.get_bytes() == b"[42.0]"
# Dynamic content type
assert d.content_type == 'json'
assert d.load() == [42.0]
def test_cpp_hello_file(test_env):
test_env.start(1, executor="cpptester")
with test_env.client.new_session() as s:
d1 = blob("WORLD")
t0 = tasks.execute("ls",
input_paths=[Input("d1", dataobj=d1)],
output_paths=[Output("d1")])
t1 = cpp_hello(t0.output)
t1.output.keep()
s.submit()
assert t1.output.fetch().get_bytes() == b"Hello WORLD!"
def test_hello_file(self, test_env):
self.start(test_env)
with test_env.client.new_session() as s:
d1 = blob("WORLD")
t0 = tasks.Execute("ls",
input_paths=[Input("d1", dataobj=d1)],
output_paths=[Output("d1")])
t1 = self.task_hello(t0.output)
t1.output.keep()
s.submit()
assert t1.output.fetch().get_bytes() == b"Hello WORLD!"
def test_execute_positional_output(test_env):
test_env.start(1)
with test_env.client.new_session() as s:
t0 = tasks.execute("ls /", stdout=True)
t1 = tasks.execute(("tee", Output("file")), stdin=t0, stdout="stdout")
t1.outputs["file"].keep()
t1.outputs["stdout"].keep()
s.submit()
f = t1.outputs["file"].fetch()
s = t1.outputs["stdout"].fetch()
assert f.get_bytes() == s.get_bytes()
def test_program_outputs(test_env):
"Specify program content type on spec and instantiation."
obj = ["1", 2.0, {'a': 42}]
program1 = Program(["cat", Input("i")], stdout="o")
program2 = Program(["cat", Input("i", content_type='pickle')],
stdout=Output(content_type='pickle'))
test_env.start(1)
with test_env.client.new_session() as s:
# Dynamic content-type, forgotten by cat
t1a = program1(i=pickled(obj))
t1a.output.keep()
# Static content-type by instantiation
t1b = program1(i=pickled(obj), output=Output(content_type='pickle'))
t1b.output.keep()
# No content type
t1c = program1(i=blob(pickle.dumps(obj)))
t1c.output.keep()
# Static content-type by Program spec
t2 = program2(i=pickled(obj))
t2.output.keep()
s.submit()
assert t1a.output.content_type is None
with pytest.raises(RainException):
assert t1a.output.fetch().load() == obj
assert t1a.output.fetch().get_bytes() == pickle.dumps(obj)
# TODO(gavento): Needs OutputSpec and Output merging
# assert t1b.output.fetch().load() == obj
assert t1c.output.content_type is None
with pytest.raises(RainException):
t1c.output.fetch().load()
assert t1a.output.fetch().get_bytes() == pickle.dumps(obj)
assert t2.output.fetch().load() == obj
def test_python_datainstance_write(test_env):
@remote()
def remote_fn(ctx, input1, input2):
input1.write("test1")
input2.write("test2")
with open("test1") as f:
assert f.read() == "Data 1"
with open("test1", "w") as f:
f.write("New data 1")
with open("test1") as f:
assert f.read() == "New data 1"
with open("test2/file") as f:
assert f.read() == "Data 2"
with open("test2/file", "w") as f:
f.write("New data 2")
with open("test2/file") as f:
assert f.read() == "New data 2"
os.mkdir("test2/testdir")
os.unlink("test1")
os.unlink("test2/file")
return b""
test_env.start(1)
with test_env.client.new_session() as s:
d1 = blob(b"Data 1")
os.mkdir("dir")
with open("dir/file", "w") as f:
f.write("Data 2")
d2 = directory("dir")
remote_fn(d1, d2)
s.submit()
s.wait_all()
x = tasks.Execute(
"ls",
input_paths=[Input("d1", dataobj=d1),
InputDir("d2", dataobj=d2)],
output_paths=[Output("d1"), OutputDir("d2")])
remote_fn(x.outputs["d1"], x.outputs["d2"])
s.submit()
s.wait_all()
def test_execute_with_dir(test_env):
path = os.path.join(test_env.work_dir, "test-dir")
os.mkdir(path)
os.mkdir(os.path.join(path, "a"))
os.mkdir(os.path.join(path, "a", "b"))
with open(os.path.join(path, "f.txt"), "w") as f:
f.write("Hello 1")
with open(os.path.join(path, "a", "b", "g.txt"), "w") as f:
f.write("Hello 2")
test_env.start(1)
with test_env.client.new_session() as s:
data = directory(path=path)
e = tasks.Execute("find ./mydir",
input_paths=[InputDir("mydir", dataobj=data)],
output_paths=[
Output("f", path="mydir/f.txt"),
OutputDir("a", path="mydir/a")
],
stdout=True)
e.keep_outputs()
s.submit()
s.wait_all()
result = set(
e.outputs["stdout"].fetch().get_bytes().strip().split(b"\n"))
assert b"./mydir" in result
assert b"./mydir/a" in result
assert b"./mydir/a/b" in result
assert b"./mydir/f.txt" in result
assert b"./mydir/a/b/g.txt" in result
assert e.outputs["f"].fetch().get_bytes() == b"Hello 1"
e.outputs["a"].fetch().write(os.path.join(test_env.work_dir, "result"))
assert os.path.isfile(
os.path.join(test_env.work_dir, "result", "b", "g.txt"))
def test1(
ctx
) -> (Output(encode='pickle', label='test_pickle', size_hint=0.1),
Output(content_type='text-latin2'), "out_c", "out_d"):
return (obj, b'\xe1\xb9\xef\xeb', pickle.dumps(obj2),
ctx.blob(b"[42.0]", content_type='json'))
