def _setup(self, config, temp):
CommandNode._setup(self, config, temp)
# The temp folder may contain old files:
# Remove old pipes to prevent failure at _teardown
for pipe_fname in glob.glob(os.path.join(temp, "pipe*")):
fileutils.try_remove(pipe_fname)
# ExaML refuses to overwrite old info files
fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline"))
# Resume from last checkpoint, if one such was generated
checkpoints = glob.glob(
os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*"))
if not checkpoints:
return
cache = FileStatusCache()
if not cache.are_files_outdated(self.input_files, checkpoints):
checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1]))
# FIXME: Less hacky solution to modifying AtomicCmds needed
self._command._command.append("-R")
self._command._command.append(checkpoints[-1])
else:
for fpath in checkpoints:
fileutils.try_remove(fpath)
def _setup(self, config, temp):
CommandNode._setup(self, config, temp)
# The temp folder may contain old files:
# Remove old pipes to prevent failure at _teardown
for pipe_fname in glob.glob(os.path.join(temp, "pipe*")):
fileutils.try_remove(pipe_fname)
# ExaML refuses to overwrite old info files
fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline"))
# Resume from last checkpoint, if one such was generated
checkpoints = glob.glob(os.path.join(temp,
"ExaML_binaryCheckpoint.Pypeline_*"))
if not checkpoints:
return
cache = FileStatusCache()
if not cache.are_files_outdated(self.input_files, checkpoints):
checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1]))
# FIXME: Less hacky solution to modifying AtomicCmds needed
self._command._command.append("-R")
self._command._command.append(checkpoints[-1])
else:
for fpath in checkpoints:
fileutils.try_remove(fpath)
def test_nodegraph_is_outdated__updates():
my_node = flexmock(input_files=(test_file("timestamp_a_older"),),
output_files=(test_file("timestamp_a_younger"),))
assert not NodeGraph.is_outdated(my_node, FileStatusCache())
my_node = flexmock(input_files=(test_file("timestamp_a_younger"),),
output_files=(test_file("timestamp_a_older"),))
assert NodeGraph.is_outdated(my_node, FileStatusCache())
def test_nodegraph_is_done__output_changes(temp_folder):
temp_file_1 = os.path.join(temp_folder, "file_1.txt")
temp_file_2 = os.path.join(temp_folder, "file_2.txt")
my_node = flexmock(output_files=(temp_file_1, temp_file_2))
assert not NodeGraph.is_done(my_node, FileStatusCache())
set_file_contents(temp_file_1, "foo")
assert not NodeGraph.is_done(my_node, FileStatusCache())
set_file_contents(temp_file_2, "bar")
assert NodeGraph.is_done(my_node, FileStatusCache())
def test_nodegraph_is_done__output_changes(tmp_path):
temp_file_1 = tmp_path / "file_1.txt"
temp_file_2 = tmp_path / "file_2.txt"
my_node = Mock(output_files=(temp_file_1, temp_file_2))
assert not NodeGraph.is_done(my_node, FileStatusCache())
temp_file_1.write_text("foo")
assert not NodeGraph.is_done(my_node, FileStatusCache())
temp_file_2.write_text("bar")
assert NodeGraph.is_done(my_node, FileStatusCache())
def test_nodegraph_is_outdated__updates(tmp_path):
older_file = create_test_file(_TIMESTAMP_1, tmp_path, "older_file")
younger_file = create_test_file(_TIMESTAMP_2, tmp_path, "younger_file")
my_node = Mock(
input_files=(older_file, ),
output_files=(younger_file, ),
)
assert not NodeGraph.is_outdated(my_node, FileStatusCache())
my_node = Mock(
input_files=(younger_file, ),
output_files=(older_file, ),
)
assert NodeGraph.is_outdated(my_node, FileStatusCache())
def list_output_files(self):
cache = FileStatusCache()
nodegraph = NodeGraph(self._nodes, lambda: cache)
output_files = {}
def collect_output_files(node):
state = None
if nodegraph.is_done(node, cache):
state = nodegraph.DONE
if nodegraph.is_outdated(node, cache):
state = nodegraph.OUTDATED
for filename in node.output_files:
output_files[os.path.abspath(filename)] = state
return True
self.walk_nodes(collect_output_files)
return output_files
def test_nodegraph_is_done__no_output():
cache = FileStatusCache()
node = flexmock(output_files=())
assert NodeGraph.is_done(node, cache)
def test_nodegraph_is_outdated__output_but_no_input():
my_node = flexmock(input_files=(),
output_files=_OUT_FILES)
assert not NodeGraph.is_outdated(my_node, FileStatusCache())
def test_nodegraph_is_done__subnode_not_considered(temp_folder):
temp_file = os.path.join(temp_folder, "file.txt")
subnode = flexmock(output_files=(temp_file,))
my_node = flexmock(output_files=(),
subnodes=(subnode,))
assert NodeGraph.is_done(my_node, FileStatusCache())
def test_nodegraph_is_outdated__output_but_no_input(tmp_path):
output_file = tmp_path / "file"
output_file.touch()
my_node = Mock(input_files=(), output_files=(output_file, ))
assert not NodeGraph.is_outdated(my_node, FileStatusCache())
def test_nodegraph_is_outdated__no_output():
my_node = Mock(input_files=(), output_files=())
assert not NodeGraph.is_outdated(my_node, FileStatusCache())