def run(self, fileStore):
cwljob = resolve_indirect(self.cwljob)
fillInDefaults(self.cwltool.tool["inputs"], cwljob)
inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
outdir = os.path.join(fileStore.getLocalTempDir(), "out")
tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
os.mkdir(inpdir)
os.mkdir(outdir)
os.mkdir(tmpdir)
# Copy input files out of the global file store.
index={}
adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index))
# Run the tool
opts = copy.deepcopy(self.executor_options)
# Exports temporary directory for batch systems that reset TMPDIR
os.environ["TMPDIR"] = os.path.realpath(opts.pop("tmpdir", None) or tmpdir)
output = cwltool.main.single_job_executor(self.cwltool, cwljob,
basedir=os.getcwd(),
outdir=outdir,
tmpdir=tmpdir,
tmpdir_prefix="tmp",
**opts)
cwltool.builder.adjustDirObjs(output, locToPath)
cwltool.builder.adjustFileObjs(output, locToPath)
# Copy output files into the global file store.
adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}))
return output
def run(self, fileStore):
cwljob = resolve_indirect(self.cwljob)
fillInDefaults(self.cwltool.tool["inputs"], cwljob)
inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
outdir = os.path.join(fileStore.getLocalTempDir(), "out")
tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
os.mkdir(inpdir)
os.mkdir(outdir)
os.mkdir(tmpdir)
# Copy input files out of the global file store.
index={}
adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index))
# Run the tool
output = cwltool.main.single_job_executor(self.cwltool, cwljob,
os.getcwd(), None,
outdir=outdir,
tmpdir=tmpdir,
**self.executor_options)
# Copy output files into the global file store.
adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}))
return output
def run(self, fileStore):
cwljob = resolve_indirect(self.cwljob)
fillInDefaults(self.cwltool.tool["inputs"], cwljob)
inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
outdir = os.path.join(fileStore.getLocalTempDir(), "out")
tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
os.mkdir(inpdir)
os.mkdir(outdir)
os.mkdir(tmpdir)
# Copy input files out of the global file store.
index={}
adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index))
# Run the tool
output = cwltool.main.single_job_executor(self.cwltool, cwljob,
os.getcwd(), None,
outdir=outdir,
tmpdir=tmpdir,
**self.executor_options)
# Copy output files into the global file store.
adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}))
return output
def run(self, fileStore):
cwljob = resolve_indirect(self.cwljob)
fillInDefaults(self.cwltool.tool["inputs"], cwljob)
inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
outdir = os.path.join(fileStore.getLocalTempDir(), "out")
tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
os.mkdir(inpdir)
os.mkdir(outdir)
os.mkdir(tmpdir)
# Copy input files out of the global file store, ensure path/location synchronized
index = {}
existing = {}
adjustFilesWithSecondary(
cwljob,
functools.partial(getFile,
fileStore,
inpdir,
index=index,
existing=existing))
cwltool.pathmapper.adjustFileObjs(cwljob, pathToLoc)
cwltool.pathmapper.adjustFileObjs(cwljob, addFilePartRefs)
# Run the tool
opts = copy.deepcopy(self.executor_options)
# Exports temporary directory for batch systems that reset TMPDIR
os.environ["TMPDIR"] = os.path.realpath(
opts.pop("tmpdir", None) or tmpdir)
(output, status) = cwltool.main.single_job_executor(
self.cwltool,
cwljob,
basedir=os.getcwd(),
outdir=outdir,
tmpdir=tmpdir,
tmpdir_prefix="tmp",
make_fs_access=cwltool.stdfsaccess.StdFsAccess,
**opts)
if status != "success":
raise cwltool.errors.WorkflowException(status)
cwltool.pathmapper.adjustDirObjs(output, locToPath)
cwltool.pathmapper.adjustFileObjs(output, locToPath)
cwltool.pathmapper.adjustFileObjs(
output,
functools.partial(computeFileChecksums,
cwltool.stdfsaccess.StdFsAccess(outdir)))
# Copy output files into the global file store.
adjustFiles(
output,
functools.partial(writeFile, fileStore.writeGlobalFile, {},
existing))
return output
def run(self, fileStore):
cwljob = resolve_indirect(self.cwljob)
fillInDefaults(self.cwltool.tool["inputs"], cwljob)
inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
outdir = os.path.join(fileStore.getLocalTempDir(), "out")
tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
os.mkdir(inpdir)
os.mkdir(outdir)
os.mkdir(tmpdir)
# Copy input files out of the global file store.
index = {}
adjustFilesWithSecondary(
cwljob, functools.partial(getFile, fileStore, inpdir, index=index))
# Run the tool
opts = copy.deepcopy(self.executor_options)
# Exports temporary directory for batch systems that reset TMPDIR
os.environ["TMPDIR"] = os.path.realpath(
opts.pop("tmpdir", None) or tmpdir)
output = cwltool.main.single_job_executor(self.cwltool,
cwljob,
basedir=os.getcwd(),
outdir=outdir,
tmpdir=tmpdir,
tmpdir_prefix="tmp",
**opts)
cwltool.builder.adjustDirObjs(output, locToPath)
cwltool.builder.adjustFileObjs(output, locToPath)
cwltool.builder.adjustFileObjs(
output,
functools.partial(computeFileChecksums,
cwltool.stdfsaccess.StdFsAccess(outdir)))
# Copy output files into the global file store.
adjustFiles(
output, functools.partial(writeFile, fileStore.writeGlobalFile,
{}))
return output
def run(self, fileStore):
cwljob = resolve_indirect(self.cwljob)
fillInDefaults(self.cwltool.tool["inputs"], cwljob)
inpdir = os.path.join(fileStore.getLocalTempDir(), "inp")
outdir = os.path.join(fileStore.getLocalTempDir(), "out")
tmpdir = os.path.join(fileStore.getLocalTempDir(), "tmp")
os.mkdir(inpdir)
os.mkdir(outdir)
os.mkdir(tmpdir)
# Copy input files out of the global file store, ensure path/location synchronized
index = {}
existing = {}
adjustFilesWithSecondary(cwljob, functools.partial(getFile, fileStore, inpdir, index=index,
existing=existing))
cwltool.pathmapper.adjustFileObjs(cwljob, pathToLoc)
cwltool.pathmapper.adjustFileObjs(cwljob, addFilePartRefs)
# Run the tool
opts = copy.deepcopy(self.executor_options)
# Exports temporary directory for batch systems that reset TMPDIR
os.environ["TMPDIR"] = os.path.realpath(opts.pop("tmpdir", None) or tmpdir)
(output, status) = cwltool.main.single_job_executor(self.cwltool, cwljob,
basedir=os.getcwd(),
outdir=outdir,
tmpdir=tmpdir,
tmpdir_prefix="tmp",
make_fs_access=cwltool.stdfsaccess.StdFsAccess,
**opts)
if status != "success":
raise cwltool.errors.WorkflowException(status)
cwltool.pathmapper.adjustDirObjs(output, locToPath)
cwltool.pathmapper.adjustFileObjs(output, locToPath)
cwltool.pathmapper.adjustFileObjs(output, functools.partial(computeFileChecksums,
cwltool.stdfsaccess.StdFsAccess(outdir)))
# Copy output files into the global file store.
adjustFiles(output, functools.partial(writeFile, fileStore.writeGlobalFile, {}, existing))
return output
def main(args=None, stdout=sys.stdout):
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
parser.add_argument("cwltool", type=str)
parser.add_argument("cwljob", type=str, nargs="?", default=None)
# Will override the "jobStore" positional argument, enables
# user to select jobStore or get a default from logic one below.
parser.add_argument("--jobStore", type=str)
parser.add_argument("--conformance-test", action="store_true")
parser.add_argument("--no-container", action="store_true")
parser.add_argument("--quiet",
dest="logLevel",
action="store_const",
const="ERROR")
parser.add_argument("--basedir", type=str)
parser.add_argument("--outdir", type=str, default=os.getcwd())
parser.add_argument("--version", action='version', version=version)
parser.add_argument(
"--preserve-environment",
type=str,
nargs='+',
help=
"Preserve specified environment variables when running CommandLineTools",
metavar=("VAR1,VAR2"),
default=("PATH", ),
dest="preserve_environment")
# mkdtemp actually creates the directory, but
# toil requires that the directory not exist,
# so make it and delete it and allow
# toil to create it again (!)
workdir = tempfile.mkdtemp()
os.rmdir(workdir)
if args is None:
args = sys.argv[1:]
options = parser.parse_args([workdir] + args)
use_container = not options.no_container
setLoggingFromOptions(options)
if options.logLevel:
cwllogger.setLevel(options.logLevel)
try:
t = cwltool.load_tool.load_tool(options.cwltool,
cwltool.workflow.defaultMakeTool)
except cwltool.process.UnsupportedRequirement as e:
logging.error(e)
return 33
if options.conformance_test:
loader = schema_salad.ref_resolver.Loader({})
else:
jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = schema_salad.ref_resolver.Loader(jobloaderctx)
if options.cwljob:
uri = (options.cwljob if urlparse.urlparse(options.cwljob).scheme else
"file://" + os.path.abspath(options.cwljob))
job, _ = loader.resolve_ref(uri, checklinks=False)
else:
job = {}
def unsupportedCheck(p):
"""Check for file inputs we don't current support in Toil:
- Directories
- File literals
"""
if p.get("class") == "Directory":
raise cwltool.process.UnsupportedRequirement(
"CWL Directory inputs not yet supported in Toil")
if p.get("contents") and (not p.get("path") and not p.get("location")):
raise cwltool.process.UnsupportedRequirement(
"CWL File literals not yet supported in Toil")
try:
cwltool.builder.adjustDirObjs(job, unsupportedCheck)
cwltool.builder.adjustFileObjs(job, unsupportedCheck)
except cwltool.process.UnsupportedRequirement as e:
logging.error(e)
return 33
cwltool.builder.adjustDirObjs(job, pathToLoc)
cwltool.builder.adjustFileObjs(job, pathToLoc)
if type(t) == int:
return t
fillInDefaults(t.tool["inputs"], job)
if options.conformance_test:
adjustFiles(job, lambda x: x.replace("file://", ""))
stdout.write(
json.dumps(cwltool.main.single_job_executor(
t,
job,
basedir=options.basedir,
tmpdir_prefix="tmp",
conformance_test=True,
use_container=use_container,
preserve_environment=options.preserve_environment),
indent=4))
return 0
if not options.basedir:
options.basedir = os.path.dirname(
os.path.abspath(options.cwljob or options.cwltool))
outdir = options.outdir
with Toil(options) as toil:
def importDefault(tool):
cwltool.builder.adjustDirObjs(tool, locToPath)
cwltool.builder.adjustFileObjs(tool, locToPath)
adjustFiles(
tool, lambda x: "file://%s" % x
if not urlparse.urlparse(x).scheme else x)
adjustFiles(tool, functools.partial(writeFile, toil.importFile,
{}))
t.visit(importDefault)
basedir = os.path.dirname(
os.path.abspath(options.cwljob or options.cwltool))
builder = t._init_job(job, basedir=basedir)
(wf1, wf2) = makeJob(t, {},
use_container=use_container,
preserve_environment=options.preserve_environment,
tmpdir=os.path.realpath(outdir))
cwltool.builder.adjustDirObjs(builder.job, locToPath)
cwltool.builder.adjustFileObjs(builder.job, locToPath)
adjustFiles(
builder.job,
lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x))
if not urlparse.urlparse(x).scheme else x)
cwltool.builder.adjustDirObjs(builder.job, pathToLoc)
cwltool.builder.adjustFileObjs(builder.job, pathToLoc)
cwltool.builder.adjustFileObjs(builder.job, addFilePartRefs)
adjustFiles(builder.job,
functools.partial(writeFile, toil.importFile, {}))
wf1.cwljob = builder.job
outobj = toil.start(wf1)
outobj = resolve_indirect(outobj)
adjustFilesWithSecondary(
outobj,
functools.partial(getFile,
toil,
outdir,
index={},
export=True,
rename_collision=True))
stdout.write(json.dumps(outobj, indent=4))
return 0
def main(args=None, stdout=sys.stdout):
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
parser.add_argument("cwltool", type=str)
parser.add_argument("cwljob", type=str, nargs="?", default=None)
# Will override the "jobStore" positional argument, enables
# user to select jobStore or get a default from logic one below.
parser.add_argument("--jobStore", type=str)
parser.add_argument("--conformance-test", action="store_true")
parser.add_argument("--not-strict", action="store_true")
parser.add_argument("--no-container", action="store_true")
parser.add_argument("--quiet", dest="logLevel", action="store_const", const="ERROR")
parser.add_argument("--basedir", type=str)
parser.add_argument("--outdir", type=str, default=os.getcwd())
parser.add_argument("--version", action='version', version=baseVersion)
parser.add_argument("--preserve-environment", type=str, nargs='+',
help="Preserve specified environment variables when running CommandLineTools",
metavar=("VAR1 VAR2"),
default=("PATH",),
dest="preserve_environment")
# mkdtemp actually creates the directory, but
# toil requires that the directory not exist,
# so make it and delete it and allow
# toil to create it again (!)
workdir = tempfile.mkdtemp()
os.rmdir(workdir)
if args is None:
args = sys.argv[1:]
options = parser.parse_args([workdir] + args)
use_container = not options.no_container
setLoggingFromOptions(options)
if options.logLevel:
cwllogger.setLevel(options.logLevel)
useStrict = not options.not_strict
try:
t = cwltool.load_tool.load_tool(options.cwltool, cwltool.workflow.defaultMakeTool,
resolver=cwltool.resolver.tool_resolver, strict=useStrict)
unsupportedRequirementsCheck(t.requirements)
except cwltool.process.UnsupportedRequirement as e:
logging.error(e)
return 33
if options.conformance_test:
loader = schema_salad.ref_resolver.Loader({})
else:
jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = schema_salad.ref_resolver.Loader(jobloaderctx)
if options.cwljob:
uri = (options.cwljob if urlparse.urlparse(options.cwljob).scheme
else "file://" + os.path.abspath(options.cwljob))
job, _ = loader.resolve_ref(uri, checklinks=False)
else:
job = {}
try:
cwltool.pathmapper.adjustDirObjs(job, unsupportedInputCheck)
cwltool.pathmapper.adjustFileObjs(job, unsupportedInputCheck)
except cwltool.process.UnsupportedRequirement as e:
logging.error(e)
return 33
cwltool.pathmapper.adjustDirObjs(job, pathToLoc)
cwltool.pathmapper.adjustFileObjs(job, pathToLoc)
if type(t) == int:
return t
fillInDefaults(t.tool["inputs"], job)
if options.conformance_test:
adjustFiles(job, lambda x: x.replace("file://", ""))
stdout.write(json.dumps(
cwltool.main.single_job_executor(t, job, basedir=options.basedir,
tmpdir_prefix="tmp",
conformance_test=True, use_container=use_container,
preserve_environment=options.preserve_environment), indent=4))
return 0
if not options.basedir:
options.basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool))
outdir = options.outdir
with Toil(options) as toil:
def importDefault(tool):
cwltool.pathmapper.adjustDirObjs(tool, locToPath)
cwltool.pathmapper.adjustFileObjs(tool, locToPath)
adjustFiles(tool, lambda x: "file://%s" % x if not urlparse.urlparse(x).scheme else x)
adjustFiles(tool, functools.partial(writeFile, toil.importFile, {}, {}))
t.visit(importDefault)
if options.restart:
outobj = toil.restart()
else:
basedir = os.path.dirname(os.path.abspath(options.cwljob or options.cwltool))
builder = t._init_job(job, basedir=basedir, use_container=use_container)
(wf1, wf2) = makeJob(t, {}, use_container=use_container,
preserve_environment=options.preserve_environment,
tmpdir=os.path.realpath(outdir), builder=builder)
try:
if isinstance(wf1, CWLWorkflow):
[unsupportedDefaultCheck(s.tool) for s in wf1.cwlwf.steps]
except cwltool.process.UnsupportedRequirement as e:
logging.error(e)
return 33
cwltool.pathmapper.adjustDirObjs(builder.job, locToPath)
cwltool.pathmapper.adjustFileObjs(builder.job, locToPath)
adjustFiles(builder.job, lambda x: "file://%s" % os.path.abspath(os.path.join(basedir, x))
if not urlparse.urlparse(x).scheme else x)
cwltool.pathmapper.adjustDirObjs(builder.job, pathToLoc)
cwltool.pathmapper.adjustFileObjs(builder.job, pathToLoc)
cwltool.pathmapper.adjustFileObjs(builder.job, addFilePartRefs)
adjustFiles(builder.job, functools.partial(writeFile, toil.importFile, {}, {}))
wf1.cwljob = builder.job
outobj = toil.start(wf1)
outobj = resolve_indirect(outobj)
try:
adjustFilesWithSecondary(outobj, functools.partial(getFile, toil, outdir, index={}, existing={},
export=True, rename_collision=True))
cwltool.pathmapper.adjustFileObjs(outobj, pathToLoc)
except cwltool.process.UnsupportedRequirement as e:
logging.error(e)
return 33
stdout.write(json.dumps(outobj, indent=4))
return 0
