def pipelineCompress(cmds, outFile):
"""execute the pipeline commands, which must write to stdout, optionally
compressing based on extension of outFile. cmds can be a single command
as a list or a list of lists". Create outFile atomically"""
if isinstance(cmds[0], str):
cmds = [cmds]
outFileTmp = fileOps.atomicTmpFile(outFile)
procOps.runProc(cmds + [[getCompressCmd(outFile)]], stdout=outFileTmp)
fileOps.atomicInstall(outFileTmp, outFile)
def atomic_install(self, target):
"""
Atomically install a given target to this task's output.
"""
output = self.output()
output.makedirs()
if isinstance(target, luigi.LocalTarget):
atomicInstall(target.path, output.path)
elif isinstance(target, str):
atomicInstall(target, output.path)
else:
raise NotImplementedError
def finishSucceed(self):
"finish production with atomic install of new output file as actual file"
self.done()
if self.installed:
raise ExRunException("output file already installed: " + self.path)
if self.outPath == None:
raise ExRunException("getOutPath() never called for: " + self.path)
if not os.path.exists(self.outPath):
raise ExRunException("output file as not created: " + self.outPath)
fileOps.atomicInstall(self.outPath, self.path)
self.installed = True
self.outPath = None
def finishSucceed(self):
"finish production with atomic install of new output file as actual file"
self.done()
if self.installed:
raise ExRunException("output file already installed: " + self.path)
if self.outPath == None:
raise ExRunException("getOutPath() never called for: " + self.path)
if not os.path.exists(self.outPath):
raise ExRunException("output file as not created: " + self.outPath)
fileOps.atomicInstall(self.outPath, self.path)
self.installed = True
self.outPath = None
def run_cmd(self, cmd, tmp_files):
"""
Run a external command that will produce the output file for this task to many files.
These files will be atomically installed.
"""
runProc(cmd)
for tmp_f, f in zip(*(tmp_files, self.output())):
f.makedirs()
if isinstance(tmp_f, luigi.LocalTarget):
atomicInstall(tmp_f.path, f.path)
elif isinstance(tmp_f, str):
atomicInstall(tmp_f, f.path)
else:
raise NotImplementedError
