def commit(self, temp): if not self.ready(): raise CmdError("Attempting to commit before command has completed") elif self._running: raise CmdError("Called 'commit' before calling 'join'") elif not os.path.samefile(self._temp, temp): raise CmdError("Mismatch between previous and current temp folders" ": %r != %s" % (self._temp, temp)) missing_files = self.expected_temp_files - set(os.listdir(temp)) if missing_files: raise CmdError("Expected files not created: %s" % (", ".join(missing_files))) temp = os.path.abspath(temp) filenames = self._generate_filenames(self._files, temp) for (key, filename) in filenames.iteritems(): if isinstance(filename, types.StringTypes): if key.startswith("OUT_"): fileutils.move_file(filename, self._files[key]) elif key.startswith("TEMP_OUT_"): fileutils.try_remove(filename) self._proc = None self._temp = None
def commit(self, temp): if not self.ready(): raise CmdError( "Attempting to commit command before it has completed") elif self._handles: raise CmdError("Called 'commit' before calling 'join'") elif not os.path.samefile(self._temp, temp): raise CmdError("Mismatch between previous and current temp folders: %r != %s" \ % (self._temp, temp)) missing_files = self.expected_temp_files - set(os.listdir(temp)) if missing_files: raise CmdError("Expected files not created: %s" % (", ".join(missing_files))) temp = os.path.abspath(temp) for (key, filename) in self._generate_filenames(self._files, temp).iteritems(): if isinstance(filename, types.StringTypes): if key.startswith("OUT_"): fileutils.move_file(filename, self._files[key]) elif key.startswith("TEMP_OUT_"): fileutils.try_remove(filename) self._proc = None self._temp = None
def commit(self, temp): committed_files = set() try: for command in self._commands: command.commit(temp) committed_files.update(command.output_files) except: # Cleanup after failed commit for fpath in committed_files: try_remove(fpath) raise
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # The temp folder may contain old files: # Remove old pipes to prevent failure at _teardown for pipe_fname in glob.glob(os.path.join(temp, "pipe*")): fileutils.try_remove(pipe_fname) # ExaML refuses to overwrite old info files fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline")) # Resume from last checkpoint, if one such was generated checkpoints = glob.glob(os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*")) checkpoints.sort(key = lambda fname: int(fname.rsplit("_", 1)[-1])) if checkpoints: # FIXME: Less hacky solution to modifying AtomicCmds needed self._command._command.append("-R") self._command._command.append(checkpoints[-1])
def _setup(self, config, temp): CommandNode._setup(self, config, temp) # The temp folder may contain old files: # Remove old pipes to prevent failure at _teardown for pipe_fname in glob.glob(os.path.join(temp, "pipe*")): fileutils.try_remove(pipe_fname) # ExaML refuses to overwrite old info files fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline")) # Resume from last checkpoint, if one such was generated checkpoints = glob.glob( os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*")) checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1])) if checkpoints: # FIXME: Less hacky solution to modifying AtomicCmds needed self._command._command.append("-R") self._command._command.append(checkpoints[-1])
def _teardown(self, config, temp_root): pipe_fname = os.path.join(temp_root, self._bam_input.pipe) os.remove(pipe_fname) try_remove(pipe_fname + ".bai") CommandNode._teardown(self, config, temp_root)
def test_try_remove__non_file(temp_folder): try_remove(temp_folder)
def test_try_remove__missing(temp_folder): fpath = os.path.join(temp_folder, "test.txt") assert not try_remove(fpath)
def test_try_remove(temp_folder): fpath = os.path.join(temp_folder, "test.txt") set_file_contents(fpath, "1 2 3") assert try_remove(fpath)