def test_atomiccmd__commit_while_running(tmp_path):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(tmp_path)
with pytest.raises(CmdError):
cmd.commit(tmp_path)
cmd.terminate()
cmd.join()
def test_atomiccmd__commit_temp_only(tmp_path):
cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt")
cmd.run(tmp_path)
assert cmd.join() == [0]
assert os.path.exists(os.path.join(tmp_path, "bar.txt"))
cmd.commit(tmp_path)
assert os.listdir(tmp_path) == []
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def test_atomiccmd__commit_before_join(tmp_path):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(tmp_path)
while cmd._proc.poll() is None:
pass
with pytest.raises(CmdError):
cmd.commit(tmp_path)
cmd.join()
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"),
TEMP_OUT_STDOUT="bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_atomiccmd__commit_temp_out(tmp_path):
dest, temp = _setup_for_commit(tmp_path, create_cmd=False)
cmd = AtomicCmd(
("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt",
)
cmd.run(temp)
assert cmd.join() == [0]
(temp / "bar.txt").write_text("1 2 3")
cmd.commit(temp)
assert os.listdir(temp) == []
assert os.listdir(dest) == ["foo.txt"]
def test_atomiccmd__commit_missing_files(tmp_path):
destination, tmp_path = _setup_for_commit(tmp_path, False)
cmd = AtomicCmd(
("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"),
OUT_BAR=os.path.join(destination, "4567"),
)
cmd.run(tmp_path)
cmd.join()
before = set(os.listdir(tmp_path))
with pytest.raises(CmdError):
cmd.commit(tmp_path)
assert before == set(os.listdir(tmp_path))
def test_atomiccmd__cleanup_proc__commit(tmp_path):
assert paleomix.atomiccmd.command._PROCS == set()
cmd = AtomicCmd("ls")
cmd.run(tmp_path)
ref = next(iter(paleomix.atomiccmd.command._PROCS))
assert ref() == cmd._proc
assert cmd.join() == [0]
# Commit frees proc object
cmd.commit(tmp_path)
assert ref() is None
assert ref not in paleomix.atomiccmd.command._PROCS
def test_atomicsets__commit(cls):
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.commit = mock.commit_1
cmd_2 = AtomicCmd(["ls"])
cmd_2.commit = mock.commit_2
cmd_3 = AtomicCmd(["ls"])
cmd_3.commit = mock.commit_3
cls((cmd_1, cmd_2, cmd_3)).commit("xTMPx")
assert mock.mock_calls == [
call.commit_1("xTMPx"),
call.commit_2("xTMPx"),
call.commit_3("xTMPx"),
]
def test_atomiccmd__commit_with_pipes(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
command_1 = AtomicCmd(("echo", "Hello, World!"), OUT_STDOUT=AtomicCmd.PIPE)
command_2 = AtomicCmd(("gzip", ),
IN_STDIN=command_1,
OUT_STDOUT=os.path.join(destination, "foo.gz"))
command_1.run(temp_folder)
command_2.run(temp_folder)
assert_equal(command_1.join(), [0])
assert_equal(command_2.join(), [0])
command_1.commit(temp_folder)
command_2.commit(temp_folder)
assert_equal(set(os.listdir(destination)), set(("foo.gz", )))
assert_equal(set(os.listdir(temp_folder)), set())
def test_atomiccmd__commit_with_pipes(tmp_path):
destination, tmp_path = _setup_for_commit(tmp_path, False)
command_1 = AtomicCmd(("echo", "Hello, World!"), OUT_STDOUT=AtomicCmd.PIPE)
command_2 = AtomicCmd(("gzip", ),
IN_STDIN=command_1,
OUT_STDOUT=os.path.join(destination, "foo.gz"))
command_1.run(tmp_path)
command_2.run(tmp_path)
assert command_1.join() == [0]
assert command_2.join() == [0]
command_1.commit(tmp_path)
command_2.commit(tmp_path)
assert set(os.listdir(destination)) == set(("foo.gz", ))
assert set(os.listdir(tmp_path)) == set()
def test_atomiccmd__commit_with_pipes(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
command_1 = AtomicCmd(("echo", "Hello, World!"),
OUT_STDOUT=AtomicCmd.PIPE)
command_2 = AtomicCmd(("gzip",),
IN_STDIN=command_1,
OUT_STDOUT=os.path.join(destination, "foo.gz"))
command_1.run(temp_folder)
command_2.run(temp_folder)
assert_equal(command_1.join(), [0])
assert_equal(command_2.join(), [0])
command_1.commit(temp_folder)
command_2.commit(temp_folder)
assert_equal(set(os.listdir(destination)), set(("foo.gz",)))
assert_equal(set(os.listdir(temp_folder)), set())
def test_atomicsets__commit__remove_files_on_failure(tmp_path, cls):
(tmp_path / "tmp").mkdir()
out_path = tmp_path / "out"
cmd_1 = AtomicCmd(["touch", "%(OUT_FILE)s"],
OUT_FILE=str(out_path / "file1"))
cmd_2 = AtomicCmd(["touch", "%(OUT_FILE)s"],
OUT_FILE=str(out_path / "file2"))
cmd_2.commit = Mock()
cmd_2.commit.side_effect = OSError()
cmdset = cls((cmd_1, cmd_2))
cmdset.run(str(tmp_path / "tmp"))
assert cmdset.join() == [0, 0]
with pytest.raises(OSError):
cmdset.commit(tmp_path / "tmp")
tmp_files = [it.name for it in (tmp_path / "tmp").iterdir()]
assert "file1" not in tmp_files
assert "file2" in tmp_files
assert list((tmp_path / "out").iterdir()) == []
def commit(self, temp):
AtomicCmd.commit(self, temp)
os.remove(os.path.join(destination, "foo.txt"))
def commit(self, temp):
AtomicCmd.commit(self, temp)
os.remove(os.path.join(destination, "foo.txt"))
def test_atomiccmd__commit_before_run():
cmd = AtomicCmd("true")
with pytest.raises(CmdError):
cmd.commit("/tmp")