def run(self, fileStore): cwljob = resolve_indirect(self.cwljob) fillInDefaults(self.cwltool.tool["inputs"], cwljob) inpdir = os.path.join(fileStore.getLocalTempDir(), "inp") outdir = os.path.join(fileStore.getLocalTempDir(), "out") tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp") os.mkdir(inpdir) os.mkdir(outdir) os.mkdir(tmpdir) # Copy input files out of the global file store. index={} adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index)) # Run the tool opts = copy.deepcopy(self.executor_options) # Exports temporary directory for batch systems that reset TMPDIR os.environ["TMPDIR"] = os.path.realpath(opts.pop("tmpdir", None) or tmpdir) output = cwltool.main.single_job_executor(self.cwltool, cwljob, basedir=os.getcwd(), outdir=outdir, tmpdir=tmpdir, tmpdir_prefix="tmp", **opts) cwltool.builder.adjustDirObjs(output, locToPath) cwltool.builder.adjustFileObjs(output, locToPath) # Copy output files into the global file store. adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {})) return output
def run(self, fileStore): cwljob = resolve_indirect(self.cwljob) fillInDefaults(self.cwltool.tool["inputs"], cwljob) inpdir = os.path.join(fileStore.getLocalTempDir(), "inp") outdir = os.path.join(fileStore.getLocalTempDir(), "out") tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp") os.mkdir(inpdir) os.mkdir(outdir) os.mkdir(tmpdir) # Copy input files out of the global file store. index={} adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index)) # Run the tool output = cwltool.main.single_job_executor(self.cwltool, cwljob, os.getcwd(), None, outdir=outdir, tmpdir=tmpdir, **self.executor_options) # Copy output files into the global file store. adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {})) return output
def run(self, fileStore): cwljob = resolve_indirect(self.cwljob) fillInDefaults(self.cwltool.tool["inputs"], cwljob) inpdir = os.path.join(fileStore.getLocalTempDir(), "inp") outdir = os.path.join(fileStore.getLocalTempDir(), "out") tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp") os.mkdir(inpdir) os.mkdir(outdir) os.mkdir(tmpdir) # Copy input files out of the global file store, ensure path/location synchronized index = {} existing = {} adjustFilesWithSecondary( cwljob, functools.partial(getFile, fileStore, inpdir, index=index, existing=existing)) cwltool.pathmapper.adjustFileObjs(cwljob, pathToLoc) cwltool.pathmapper.adjustFileObjs(cwljob, addFilePartRefs) # Run the tool opts = copy.deepcopy(self.executor_options) # Exports temporary directory for batch systems that reset TMPDIR os.environ["TMPDIR"] = os.path.realpath( opts.pop("tmpdir", None) or tmpdir) (output, status) = cwltool.main.single_job_executor( self.cwltool, cwljob, basedir=os.getcwd(), outdir=outdir, tmpdir=tmpdir, tmpdir_prefix="tmp", make_fs_access=cwltool.stdfsaccess.StdFsAccess, **opts) if status != "success": raise cwltool.errors.WorkflowException(status) cwltool.pathmapper.adjustDirObjs(output, locToPath) cwltool.pathmapper.adjustFileObjs(output, locToPath) cwltool.pathmapper.adjustFileObjs( output, functools.partial(computeFileChecksums, cwltool.stdfsaccess.StdFsAccess(outdir))) # Copy output files into the global file store. adjustFiles( output, functools.partial(writeFile, fileStore.writeGlobalFile, {}, existing)) return output
def run(self, fileStore): cwljob = resolve_indirect(self.cwljob) fillInDefaults(self.cwltool.tool["inputs"], cwljob) inpdir = os.path.join(fileStore.getLocalTempDir(), "inp") outdir = os.path.join(fileStore.getLocalTempDir(), "out") tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp") os.mkdir(inpdir) os.mkdir(outdir) os.mkdir(tmpdir) # Copy input files out of the global file store. index = {} adjustFilesWithSecondary( cwljob, functools.partial(getFile, fileStore, inpdir, index=index)) # Run the tool opts = copy.deepcopy(self.executor_options) # Exports temporary directory for batch systems that reset TMPDIR os.environ["TMPDIR"] = os.path.realpath( opts.pop("tmpdir", None) or tmpdir) output = cwltool.main.single_job_executor(self.cwltool, cwljob, basedir=os.getcwd(), outdir=outdir, tmpdir=tmpdir, tmpdir_prefix="tmp", **opts) cwltool.builder.adjustDirObjs(output, locToPath) cwltool.builder.adjustFileObjs(output, locToPath) cwltool.builder.adjustFileObjs( output, functools.partial(computeFileChecksums, cwltool.stdfsaccess.StdFsAccess(outdir))) # Copy output files into the global file store. adjustFiles( output, functools.partial(writeFile, fileStore.writeGlobalFile, {})) return output
def run(self, fileStore): cwljob = resolve_indirect(self.cwljob) fillInDefaults(self.cwltool.tool["inputs"], cwljob) inpdir = os.path.join(fileStore.getLocalTempDir(), "inp") outdir = os.path.join(fileStore.getLocalTempDir(), "out") tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp") os.mkdir(inpdir) os.mkdir(outdir) os.mkdir(tmpdir) # Copy input files out of the global file store, ensure path/location synchronized index = {} existing = {} adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index, existing=existing)) cwltool.pathmapper.adjustFileObjs(cwljob, pathToLoc) cwltool.pathmapper.adjustFileObjs(cwljob, addFilePartRefs) # Run the tool opts = copy.deepcopy(self.executor_options) # Exports temporary directory for batch systems that reset TMPDIR os.environ["TMPDIR"] = os.path.realpath(opts.pop("tmpdir", None) or tmpdir) (output, status) = cwltool.main.single_job_executor(self.cwltool, cwljob, basedir=os.getcwd(), outdir=outdir, tmpdir=tmpdir, tmpdir_prefix="tmp", make_fs_access=cwltool.stdfsaccess.StdFsAccess, **opts) if status != "success": raise cwltool.errors.WorkflowException(status) cwltool.pathmapper.adjustDirObjs(output, locToPath) cwltool.pathmapper.adjustFileObjs(output, locToPath) cwltool.pathmapper.adjustFileObjs(output, functools.partial(computeFileChecksums, cwltool.stdfsaccess.StdFsAccess(outdir))) # Copy output files into the global file store. adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}, existing)) return output
def main(args=None, stdout=sys.stdout): parser = ArgumentParser() Job.Runner.addToilOptions(parser) parser.add_argument("cwltool", type=str) parser.add_argument("cwljob", type=str, nargs="?", default=None) # Will override the "jobStore" positional argument, enables # user to select jobStore or get a default from logic one below. parser.add_argument("--jobStore", type=str) parser.add_argument("--conformance-test", action="store_true") parser.add_argument("--no-container", action="store_true") parser.add_argument("--quiet", dest="logLevel", action="store_const", const="ERROR") parser.add_argument("--basedir", type=str) parser.add_argument("--outdir", type=str, default=os.getcwd()) parser.add_argument("--version", action='version', version=version) parser.add_argument( "--preserve-environment", type=str, nargs='+', help= "Preserve specified environment variables when running CommandLineTools", metavar=("VAR1,VAR2"), default=("PATH", ), dest="preserve_environment") # mkdtemp actually creates the directory, but # toil requires that the directory not exist, # so make it and delete it and allow # toil to create it again (!) workdir = tempfile.mkdtemp() os.rmdir(workdir) if args is None: args = sys.argv[1:] options = parser.parse_args([workdir] + args) use_container = not options.no_container setLoggingFromOptions(options) if options.logLevel: cwllogger.setLevel(options.logLevel) try: t = cwltool.load_tool.load_tool(options.cwltool, cwltool.workflow.defaultMakeTool) except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 if options.conformance_test: loader = schema_salad.ref_resolver.Loader({}) else: jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = schema_salad.ref_resolver.Loader(jobloaderctx) if options.cwljob: uri = (options.cwljob if urlparse.urlparse(options.cwljob).scheme else "file://" + os.path.abspath(options.cwljob)) job, _ = loader.resolve_ref(uri, checklinks=False) else: job = {} def unsupportedCheck(p): """Check for file inputs we don't current support in Toil: - Directories - File literals """ if p.get("class") == "Directory": raise cwltool.process.UnsupportedRequirement( "CWL Directory inputs not yet supported in Toil") if p.get("contents") and (not p.get("path") and not p.get("location")): raise cwltool.process.UnsupportedRequirement( "CWL File literals not yet supported in Toil") try: cwltool.builder.adjustDirObjs(job, unsupportedCheck) cwltool.builder.adjustFileObjs(job, unsupportedCheck) except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 cwltool.builder.adjustDirObjs(job, pathToLoc) cwltool.builder.adjustFileObjs(job, pathToLoc) if type(t) == int: return t fillInDefaults(t.tool["inputs"], job) if options.conformance_test: adjustFiles(job, lambda x: x.replace("file://", "")) stdout.write( json.dumps(cwltool.main.single_job_executor( t, job, basedir=options.basedir, tmpdir_prefix="tmp", conformance_test=True, use_container=use_container, preserve_environment=options.preserve_environment), indent=4)) return 0 if not options.basedir: options.basedir = os.path.dirname( os.path.abspath(options.cwljob or options.cwltool)) outdir = options.outdir with Toil(options) as toil: def importDefault(tool): cwltool.builder.adjustDirObjs(tool, locToPath) cwltool.builder.adjustFileObjs(tool, locToPath) adjustFiles( tool, lambda x: "file://%s" % x if not urlparse.urlparse(x).scheme else x) adjustFiles(tool, functools.partial(writeFile, toil.importFile, {})) t.visit(importDefault) basedir = os.path.dirname( os.path.abspath(options.cwljob or options.cwltool)) builder = t._init_job(job, basedir=basedir) (wf1, wf2) = makeJob(t, {}, use_container=use_container, preserve_environment=options.preserve_environment, tmpdir=os.path.realpath(outdir)) cwltool.builder.adjustDirObjs(builder.job, locToPath) cwltool.builder.adjustFileObjs(builder.job, locToPath) adjustFiles( builder.job, lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x)) if not urlparse.urlparse(x).scheme else x) cwltool.builder.adjustDirObjs(builder.job, pathToLoc) cwltool.builder.adjustFileObjs(builder.job, pathToLoc) cwltool.builder.adjustFileObjs(builder.job, addFilePartRefs) adjustFiles(builder.job, functools.partial(writeFile, toil.importFile, {})) wf1.cwljob = builder.job outobj = toil.start(wf1) outobj = resolve_indirect(outobj) adjustFilesWithSecondary( outobj, functools.partial(getFile, toil, outdir, index={}, export=True, rename_collision=True)) stdout.write(json.dumps(outobj, indent=4)) return 0
def main(args=None, stdout=sys.stdout): parser = ArgumentParser() Job.Runner.addToilOptions(parser) parser.add_argument("cwltool", type=str) parser.add_argument("cwljob", type=str, nargs="?", default=None) # Will override the "jobStore" positional argument, enables # user to select jobStore or get a default from logic one below. parser.add_argument("--jobStore", type=str) parser.add_argument("--conformance-test", action="store_true") parser.add_argument("--not-strict", action="store_true") parser.add_argument("--no-container", action="store_true") parser.add_argument("--quiet", dest="logLevel", action="store_const", const="ERROR") parser.add_argument("--basedir", type=str) parser.add_argument("--outdir", type=str, default=os.getcwd()) parser.add_argument("--version", action='version', version=baseVersion) parser.add_argument("--preserve-environment", type=str, nargs='+', help="Preserve specified environment variables when running CommandLineTools", metavar=("VAR1 VAR2"), default=("PATH",), dest="preserve_environment") # mkdtemp actually creates the directory, but # toil requires that the directory not exist, # so make it and delete it and allow # toil to create it again (!) workdir = tempfile.mkdtemp() os.rmdir(workdir) if args is None: args = sys.argv[1:] options = parser.parse_args([workdir] + args) use_container = not options.no_container setLoggingFromOptions(options) if options.logLevel: cwllogger.setLevel(options.logLevel) useStrict = not options.not_strict try: t = cwltool.load_tool.load_tool(options.cwltool, cwltool.workflow.defaultMakeTool, resolver=cwltool.resolver.tool_resolver, strict=useStrict) unsupportedRequirementsCheck(t.requirements) except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 if options.conformance_test: loader = schema_salad.ref_resolver.Loader({}) else: jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = schema_salad.ref_resolver.Loader(jobloaderctx) if options.cwljob: uri = (options.cwljob if urlparse.urlparse(options.cwljob).scheme else "file://" + os.path.abspath(options.cwljob)) job, _ = loader.resolve_ref(uri, checklinks=False) else: job = {} try: cwltool.pathmapper.adjustDirObjs(job, unsupportedInputCheck) cwltool.pathmapper.adjustFileObjs(job, unsupportedInputCheck) except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 cwltool.pathmapper.adjustDirObjs(job, pathToLoc) cwltool.pathmapper.adjustFileObjs(job, pathToLoc) if type(t) == int: return t fillInDefaults(t.tool["inputs"], job) if options.conformance_test: adjustFiles(job, lambda x: x.replace("file://", "")) stdout.write(json.dumps( cwltool.main.single_job_executor(t, job, basedir=options.basedir, tmpdir_prefix="tmp", conformance_test=True, use_container=use_container, preserve_environment=options.preserve_environment), indent=4)) return 0 if not options.basedir: options.basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool)) outdir = options.outdir with Toil(options) as toil: def importDefault(tool): cwltool.pathmapper.adjustDirObjs(tool, locToPath) cwltool.pathmapper.adjustFileObjs(tool, locToPath) adjustFiles(tool, lambda x: "file://%s" % x if not urlparse.urlparse(x).scheme else x) adjustFiles(tool, functools.partial(writeFile, toil.importFile, {}, {})) t.visit(importDefault) if options.restart: outobj = toil.restart() else: basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool)) builder = t._init_job(job, basedir=basedir, use_container=use_container) (wf1, wf2) = makeJob(t, {}, use_container=use_container, preserve_environment=options.preserve_environment, tmpdir=os.path.realpath(outdir), builder=builder) try: if isinstance(wf1, CWLWorkflow): [unsupportedDefaultCheck(s.tool) for s in wf1.cwlwf.steps] except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 cwltool.pathmapper.adjustDirObjs(builder.job, locToPath) cwltool.pathmapper.adjustFileObjs(builder.job, locToPath) adjustFiles(builder.job, lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x)) if not urlparse.urlparse(x).scheme else x) cwltool.pathmapper.adjustDirObjs(builder.job, pathToLoc) cwltool.pathmapper.adjustFileObjs(builder.job, pathToLoc) cwltool.pathmapper.adjustFileObjs(builder.job, addFilePartRefs) adjustFiles(builder.job, functools.partial(writeFile, toil.importFile, {}, {})) wf1.cwljob = builder.job outobj = toil.start(wf1) outobj = resolve_indirect(outobj) try: adjustFilesWithSecondary(outobj, functools.partial(getFile, toil, outdir, index={}, existing={}, export=True, rename_collision=True)) cwltool.pathmapper.adjustFileObjs(outobj, pathToLoc) except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 stdout.write(json.dumps(outobj, indent=4)) return 0