def commit(self, temp):
if not self.ready():
raise CmdError("Attempting to commit before command has completed")
elif self._running:
raise CmdError("Called 'commit' before calling 'join'")
elif not os.path.samefile(self._temp, temp):
raise CmdError("Mismatch between previous and current temp folders"
": %r != %s" % (self._temp, temp))
missing_files = self.expected_temp_files - set(os.listdir(temp))
if missing_files:
raise CmdError("Expected files not created: %s"
% (", ".join(missing_files)))
temp = os.path.abspath(temp)
filenames = self._generate_filenames(self._files, temp)
for (key, filename) in filenames.iteritems():
if isinstance(filename, types.StringTypes):
if key.startswith("OUT_"):
fileutils.move_file(filename, self._files[key])
elif key.startswith("TEMP_OUT_"):
fileutils.try_remove(filename)
self._proc = None
self._temp = None
def commit(self, temp):
if not self.ready():
raise CmdError(
"Attempting to commit command before it has completed")
elif self._handles:
raise CmdError("Called 'commit' before calling 'join'")
elif not os.path.samefile(self._temp, temp):
raise CmdError("Mismatch between previous and current temp folders: %r != %s" \
% (self._temp, temp))
missing_files = self.expected_temp_files - set(os.listdir(temp))
if missing_files:
raise CmdError("Expected files not created: %s" %
(", ".join(missing_files)))
temp = os.path.abspath(temp)
for (key, filename) in self._generate_filenames(self._files,
temp).iteritems():
if isinstance(filename, types.StringTypes):
if key.startswith("OUT_"):
fileutils.move_file(filename, self._files[key])
elif key.startswith("TEMP_OUT_"):
fileutils.try_remove(filename)
self._proc = None
self._temp = None
def commit(self, temp):
committed_files = set()
try:
for command in self._commands:
command.commit(temp)
committed_files.update(command.output_files)
except:
# Cleanup after failed commit
for fpath in committed_files:
try_remove(fpath)
raise
def commit(self, temp):
committed_files = set()
try:
for command in self._commands:
command.commit(temp)
committed_files.update(command.output_files)
except:
# Cleanup after failed commit
for fpath in committed_files:
try_remove(fpath)
raise
def _setup(self, config, temp):
CommandNode._setup(self, config, temp)
# The temp folder may contain old files:
# Remove old pipes to prevent failure at _teardown
for pipe_fname in glob.glob(os.path.join(temp, "pipe*")):
fileutils.try_remove(pipe_fname)
# ExaML refuses to overwrite old info files
fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline"))
# Resume from last checkpoint, if one such was generated
checkpoints = glob.glob(os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*"))
checkpoints.sort(key = lambda fname: int(fname.rsplit("_", 1)[-1]))
if checkpoints:
# FIXME: Less hacky solution to modifying AtomicCmds needed
self._command._command.append("-R")
self._command._command.append(checkpoints[-1])
def _setup(self, config, temp):
CommandNode._setup(self, config, temp)
# The temp folder may contain old files:
# Remove old pipes to prevent failure at _teardown
for pipe_fname in glob.glob(os.path.join(temp, "pipe*")):
fileutils.try_remove(pipe_fname)
# ExaML refuses to overwrite old info files
fileutils.try_remove(os.path.join(temp, "ExaML_info.Pypeline"))
# Resume from last checkpoint, if one such was generated
checkpoints = glob.glob(
os.path.join(temp, "ExaML_binaryCheckpoint.Pypeline_*"))
checkpoints.sort(key=lambda fname: int(fname.rsplit("_", 1)[-1]))
if checkpoints:
# FIXME: Less hacky solution to modifying AtomicCmds needed
self._command._command.append("-R")
self._command._command.append(checkpoints[-1])
def _teardown(self, config, temp_root):
pipe_fname = os.path.join(temp_root, self._bam_input.pipe)
os.remove(pipe_fname)
try_remove(pipe_fname + ".bai")
CommandNode._teardown(self, config, temp_root)
def test_try_remove__non_file(temp_folder):
try_remove(temp_folder)
def test_try_remove__missing(temp_folder):
fpath = os.path.join(temp_folder, "test.txt")
assert not try_remove(fpath)
def test_try_remove(temp_folder):
fpath = os.path.join(temp_folder, "test.txt")
set_file_contents(fpath, "1 2 3")
assert try_remove(fpath)
def test_try_remove__non_file(temp_folder):
try_remove(temp_folder)
def test_try_remove__missing(temp_folder):
fpath = os.path.join(temp_folder, "test.txt")
assert not try_remove(fpath)
def test_try_remove(temp_folder):
fpath = os.path.join(temp_folder, "test.txt")
set_file_contents(fpath, "1 2 3")
assert try_remove(fpath)
def _teardown(self, config, temp_root):
pipe_fname = os.path.join(temp_root, self._bam_input.pipe)
os.remove(pipe_fname)
try_remove(pipe_fname + ".bai")
CommandNode._teardown(self, config, temp_root)
