def test_slice_directory2(test_env):
os.mkdir("toplevel")
with open("toplevel/file1.txt", "w") as f:
f.write("My data 1")
os.mkdir("toplevel/dir1")
os.mkdir("toplevel/dir1/dir2")
with open("toplevel/dir1/dir2/file2.txt", "w") as f:
f.write("My data 2")
test_env.start(1, delete_list_timeout=0)
with test_env.client.new_session() as s:
d = directory("toplevel")
# Force fs mapping
d = tasks.Execute("ls",
input_paths=[InputDir("d", dataobj=d)],
output_paths=[OutputDir("d")])
a1 = tasks.SliceDirectory(d, "file1.txt")
a1.output.keep()
a2 = tasks.SliceDirectory(d, "dir1/")
a2.output.keep()
a3 = tasks.SliceDirectory(d, "dir1/")
a3.output.keep()
a4 = tasks.SliceDirectory(d, "dir1/dir2/file2.txt")
a4.output.keep()
s.submit()
assert b"My data 1" == a1.output.fetch().get_bytes()
a2.output.fetch().write("result2")
with open("result2/dir2/file2.txt") as f:
assert f.read() == "My data 2"
a3.output.fetch().write("result3")
with open("result2/dir2/file2.txt") as f:
assert f.read() == "My data 2"
assert b"My data 2" == a4.output.fetch().get_bytes()
def test_slice_directory1(test_env):
os.mkdir("toplevel")
with open("toplevel/file1.txt", "w") as f:
f.write("My data 1")
os.mkdir("toplevel/dir1")
os.mkdir("toplevel/dir1/dir2")
with open("toplevel/dir1/dir2/file2.txt", "w") as f:
f.write("My data 2")
test_env.start(1, delete_list_timeout=0)
with test_env.client.new_session() as s:
d = directory("toplevel")
a1 = tasks.slice_directory(d, "file1.txt")
a1.output.keep()
a2 = tasks.slice_directory(d, "dir1/")
a2.output.keep()
a3 = tasks.slice_directory(d, "dir1/")
a3.output.keep()
a4 = tasks.slice_directory(d, "dir1/dir2/file2.txt")
a4.output.keep()
s.submit()
assert b"My data 1" == a1.output.fetch().get_bytes()
a2.output.fetch().write("result2")
with open("result2/dir2/file2.txt") as f:
assert f.read() == "My data 2"
a3.output.fetch().write("result3")
with open("result2/dir2/file2.txt") as f:
assert f.read() == "My data 2"
assert b"My data 2" == a4.output.fetch().get_bytes()
def test_make_directory(test_env):
test_env.start(1, delete_list_timeout=0)
# TODO: EMPTY DIR os.mkdir("empty")
os.mkdir("mydir3")
with open("mydir3/file.txt", "w") as f:
f.write("My data 4")
with test_env.client.new_session() as s:
b1 = blob(b"My data 1")
b2 = blob(b"My data 2")
b3 = blob(b"My data 3")
d1 = directory("mydir3")
# TODO: EMPTY DIR d2 = directory("empty")
t0 = tasks.Execute(["/bin/cat", b1],
stdout=True,
input_paths=[InputDir("d1", dataobj=d1)],
output_paths=[OutputDir("d1")])
r = tasks.MakeDirectory([
("myfile1", t0.outputs["stdout"]),
("mydir/mydir2/myfile2", b2),
("mydir/myfile3", b3),
("mydir/d1a", d1),
# ("mydir/d2", d2),
("mydir/d1b", t0.outputs["d1"]),
])
r.output.keep()
s.submit()
s.wait_all()
r.output.fetch().write("rdir")
with open(os.path.join(test_env.work_dir, "rdir", "myfile1")) as f:
assert f.read() == "My data 1"
with open(
os.path.join(test_env.work_dir, "rdir", "mydir", "mydir2",
"myfile2")) as f:
assert f.read() == "My data 2"
with open(os.path.join(test_env.work_dir, "rdir", "mydir",
"myfile3")) as f:
assert f.read() == "My data 3"
with open(os.path.join(test_env.work_dir, "rdir", "mydir",
"myfile3")) as f:
assert f.read() == "My data 3"
with open(
os.path.join(test_env.work_dir, "rdir", "mydir", "d1a",
"file.txt")) as f:
assert f.read() == "My data 4"
with open(
os.path.join(test_env.work_dir, "rdir", "mydir", "d1b",
"file.txt")) as f:
assert f.read() == "My data 4"
def test_python_datainstance_write(test_env):
@remote()
def remote_fn(ctx, input1, input2):
input1.write("test1")
input2.write("test2")
with open("test1") as f:
assert f.read() == "Data 1"
with open("test1", "w") as f:
f.write("New data 1")
with open("test1") as f:
assert f.read() == "New data 1"
with open("test2/file") as f:
assert f.read() == "Data 2"
with open("test2/file", "w") as f:
f.write("New data 2")
with open("test2/file") as f:
assert f.read() == "New data 2"
os.mkdir("test2/testdir")
os.unlink("test1")
os.unlink("test2/file")
return b""
test_env.start(1)
with test_env.client.new_session() as s:
d1 = blob(b"Data 1")
os.mkdir("dir")
with open("dir/file", "w") as f:
f.write("Data 2")
d2 = directory("dir")
remote_fn(d1, d2)
s.submit()
s.wait_all()
x = tasks.Execute(
"ls",
input_paths=[Input("d1", dataobj=d1),
InputDir("d2", dataobj=d2)],
output_paths=[Output("d1"), OutputDir("d2")])
remote_fn(x.outputs["d1"], x.outputs["d2"])
s.submit()
s.wait_all()
def test_dir_big(test_env):
data = b"01234567890" * 1024 * 1024
os.mkdir("dir")
with open("dir/file1", "wb") as f:
f.write(data)
test_env.start(1)
with test_env.client.new_session() as s:
d = directory("dir")
t = tasks.Execute("cat d/file1",
input_paths=[InputDir("d", dataobj=d)],
stdout=True,
output_paths=[OutputDir("d")])
t.keep_outputs()
s.submit()
assert t.outputs["stdout"].fetch().get_bytes() == data
t.outputs["d"].fetch().write("result")
with open("result/file1", "rb") as f:
assert f.read() == data
def test_execute_with_dir(test_env):
path = os.path.join(test_env.work_dir, "test-dir")
os.mkdir(path)
os.mkdir(os.path.join(path, "a"))
os.mkdir(os.path.join(path, "a", "b"))
with open(os.path.join(path, "f.txt"), "w") as f:
f.write("Hello 1")
with open(os.path.join(path, "a", "b", "g.txt"), "w") as f:
f.write("Hello 2")
test_env.start(1)
with test_env.client.new_session() as s:
data = directory(path=path)
e = tasks.Execute("find ./mydir",
input_paths=[InputDir("mydir", dataobj=data)],
output_paths=[
Output("f", path="mydir/f.txt"),
OutputDir("a", path="mydir/a")
],
stdout=True)
e.keep_outputs()
s.submit()
s.wait_all()
result = set(
e.outputs["stdout"].fetch().get_bytes().strip().split(b"\n"))
assert b"./mydir" in result
assert b"./mydir/a" in result
assert b"./mydir/a/b" in result
assert b"./mydir/f.txt" in result
assert b"./mydir/a/b/g.txt" in result
assert e.outputs["f"].fetch().get_bytes() == b"Hello 1"
e.outputs["a"].fetch().write(os.path.join(test_env.work_dir, "result"))
assert os.path.isfile(
os.path.join(test_env.work_dir, "result", "b", "g.txt"))
def test_python_dir(test_env):
@remote(outputs=[OutputDir("output")])
def remote_fn(ctx, input1, input2):
input2.write("mydir")
input1.write("myfile")
assert os.path.isfile("myfile")
assert os.path.isdir("mydir")
with open("mydir/a/b/g.txt") as f:
assert f.read() == "Hello 2"
with open("mydir/f.txt") as f:
assert f.read() == "Hello 1"
os.mkdir("test")
ctx.stage_directory("test")
return ctx.stage_directory("mydir/a")
@remote(outputs=[OutputDir("output")])
def remote_fn2(ctx, input1):
return input1
@remote(outputs=[OutputDir("output")])
def remote_fn3(ctx, input1):
input1.write("x")
with open(os.path.join("x", "b", "g.txt"), "w") as f:
f.write("Hello 3")
os.mkdir("x/test3")
return ctx.stage_directory("x")
path = os.path.join(test_env.work_dir, "test-dir")
os.mkdir(path)
os.mkdir(os.path.join(path, "a"))
os.mkdir(os.path.join(path, "a", "b"))
with open(os.path.join(path, "f.txt"), "w") as f:
f.write("Hello 1")
with open(os.path.join(path, "a", "b", "g.txt"), "w") as f:
f.write("Hello 2")
test_env.start(2)
with test_env.client.new_session() as s:
data = directory(path=path)
e = remote_fn(blob(b"Hello1"), data)
e1 = remote_fn2(e)
e1.keep_outputs()
e2 = remote_fn3(e)
e2.keep_outputs()
s.submit()
s.wait_all()
result = e1.output.fetch()
result.write(os.path.join(test_env.work_dir, "rdir1"))
with open(os.path.join(test_env.work_dir, "rdir1", "b", "g.txt")) as f:
assert f.read() == "Hello 2"
result = e2.output.fetch()
result.write(os.path.join(test_env.work_dir, "rdir2"))
with open(os.path.join(test_env.work_dir, "rdir2", "b", "g.txt")) as f:
assert f.read() == "Hello 3"
assert os.path.isdir("rdir2/test3")
