def pipelineCompress(cmds, outFile): """execute the pipeline commands, which must write to stdout, optionally compressing based on extension of outFile. cmds can be a single command as a list or a list of lists". Create outFile atomically""" if isinstance(cmds[0], str): cmds = [cmds] outFileTmp = fileOps.atomicTmpFile(outFile) procOps.runProc(cmds + [[getCompressCmd(outFile)]], stdout=outFileTmp) fileOps.atomicInstall(outFileTmp, outFile)
def atomic_install(self, target): """ Atomically install a given target to this task's output. """ output = self.output() output.makedirs() if isinstance(target, luigi.LocalTarget): atomicInstall(target.path, output.path) elif isinstance(target, str): atomicInstall(target, output.path) else: raise NotImplementedError
def finishSucceed(self): "finish production with atomic install of new output file as actual file" self.done() if self.installed: raise ExRunException("output file already installed: " + self.path) if self.outPath == None: raise ExRunException("getOutPath() never called for: " + self.path) if not os.path.exists(self.outPath): raise ExRunException("output file as not created: " + self.outPath) fileOps.atomicInstall(self.outPath, self.path) self.installed = True self.outPath = None
def run_cmd(self, cmd, tmp_files): """ Run a external command that will produce the output file for this task to many files. These files will be atomically installed. """ runProc(cmd) for tmp_f, f in zip(*(tmp_files, self.output())): f.makedirs() if isinstance(tmp_f, luigi.LocalTarget): atomicInstall(tmp_f.path, f.path) elif isinstance(tmp_f, str): atomicInstall(tmp_f, f.path) else: raise NotImplementedError