def _setup(self, config, temp): CommandNode._setup(self, config, temp) # The temp folder may contain old files: # Remove old pipes to prevent failure at _teardown for pipe_fname in glob.glob(os.path.join(temp, "pipe*")): fileutils.try_remove(pipe_fname) # ExaML refuses to overwrite old info files fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline")) # Resume from last checkpoint, if one such was generated checkpoints = glob.glob( os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*")) if not checkpoints: return cache = FileStatusCache() if not cache.are_files_outdated(self.input_files, checkpoints): checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1])) # FIXME: Less hacky solution to modifying AtomicCmds needed self._command._command.append("-R") self._command._command.append(checkpoints[-1]) else: for fpath in checkpoints: fileutils.try_remove(fpath)
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # The temp folder may contain old files: # Remove old pipes to prevent failure at _teardown for pipe_fname in glob.glob(os.path.join(temp, "pipe*")): fileutils.try_remove(pipe_fname) # ExaML refuses to overwrite old info files fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline")) # Resume from last checkpoint, if one such was generated checkpoints = glob.glob(os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*")) if not checkpoints: return cache = FileStatusCache() if not cache.are_files_outdated(self.input_files, checkpoints): checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1])) # FIXME: Less hacky solution to modifying AtomicCmds needed self._command._command.append("-R") self._command._command.append(checkpoints[-1]) else: for fpath in checkpoints: fileutils.try_remove(fpath)
def test_nodegraph_is_outdated__updates(): my_node = flexmock(input_files=(test_file("timestamp_a_older"),), output_files=(test_file("timestamp_a_younger"),)) assert not NodeGraph.is_outdated(my_node, FileStatusCache()) my_node = flexmock(input_files=(test_file("timestamp_a_younger"),), output_files=(test_file("timestamp_a_older"),)) assert NodeGraph.is_outdated(my_node, FileStatusCache())
def test_nodegraph_is_done__output_changes(temp_folder): temp_file_1 = os.path.join(temp_folder, "file_1.txt") temp_file_2 = os.path.join(temp_folder, "file_2.txt") my_node = flexmock(output_files=(temp_file_1, temp_file_2)) assert not NodeGraph.is_done(my_node, FileStatusCache()) set_file_contents(temp_file_1, "foo") assert not NodeGraph.is_done(my_node, FileStatusCache()) set_file_contents(temp_file_2, "bar") assert NodeGraph.is_done(my_node, FileStatusCache())
def test_nodegraph_is_done__output_changes(tmp_path): temp_file_1 = tmp_path / "file_1.txt" temp_file_2 = tmp_path / "file_2.txt" my_node = Mock(output_files=(temp_file_1, temp_file_2)) assert not NodeGraph.is_done(my_node, FileStatusCache()) temp_file_1.write_text("foo") assert not NodeGraph.is_done(my_node, FileStatusCache()) temp_file_2.write_text("bar") assert NodeGraph.is_done(my_node, FileStatusCache())
def test_nodegraph_is_outdated__updates(tmp_path): older_file = create_test_file(_TIMESTAMP_1, tmp_path, "older_file") younger_file = create_test_file(_TIMESTAMP_2, tmp_path, "younger_file") my_node = Mock( input_files=(older_file, ), output_files=(younger_file, ), ) assert not NodeGraph.is_outdated(my_node, FileStatusCache()) my_node = Mock( input_files=(younger_file, ), output_files=(older_file, ), ) assert NodeGraph.is_outdated(my_node, FileStatusCache())
def list_output_files(self): cache = FileStatusCache() nodegraph = NodeGraph(self._nodes, lambda: cache) output_files = {} def collect_output_files(node): state = None if nodegraph.is_done(node, cache): state = nodegraph.DONE if nodegraph.is_outdated(node, cache): state = nodegraph.OUTDATED for filename in node.output_files: output_files[os.path.abspath(filename)] = state return True self.walk_nodes(collect_output_files) return output_files
def test_nodegraph_is_done__no_output(): cache = FileStatusCache() node = flexmock(output_files=()) assert NodeGraph.is_done(node, cache)
def test_nodegraph_is_outdated__output_but_no_input(): my_node = flexmock(input_files=(), output_files=_OUT_FILES) assert not NodeGraph.is_outdated(my_node, FileStatusCache())
def test_nodegraph_is_done__subnode_not_considered(temp_folder): temp_file = os.path.join(temp_folder, "file.txt") subnode = flexmock(output_files=(temp_file,)) my_node = flexmock(output_files=(), subnodes=(subnode,)) assert NodeGraph.is_done(my_node, FileStatusCache())
def test_nodegraph_is_outdated__output_but_no_input(tmp_path): output_file = tmp_path / "file" output_file.touch() my_node = Mock(input_files=(), output_files=(output_file, )) assert not NodeGraph.is_outdated(my_node, FileStatusCache())
def test_nodegraph_is_outdated__no_output(): my_node = Mock(input_files=(), output_files=()) assert not NodeGraph.is_outdated(my_node, FileStatusCache())