def exeval(ex, jobinput, requirements, docpath, context, pull_image):
for r in reversed(requirements):
if r["class"] == "ExpressionEngineRequirement" and r["id"] == ex[
"engine"]:
if r["id"][0] != "#":
with open(os.path.join(docpath, r["id"])) as f:
ex_obj = yaml.load(f)
sch = process.get_schema()
validate.validate_ex(
sch.get_name("ExpressionEngineRequirement", ""), ex_obj)
r = ex_obj
runtime = []
img_id = docker.get_from_requirements(r.get("requirements"),
r.get("hints"), pull_image)
if img_id:
runtime = ["docker", "run", "-i", "--rm", img_id]
exdefs = []
for exdef in r.get("expressionDefs", []):
if isinstance(exdef, dict) and "ref" in exdef:
with open(os.path.join(r["_docpath"], exdef["ref"])) as f:
exdefs.append(f.read())
elif isinstance(exdef, basestring):
exdefs.append(exdef)
inp = {
"script": ex["script"],
"expressionDefs": exdefs,
"job": jobinput,
"context": context
}
_logger.debug(json.dumps(inp))
sp = subprocess.Popen(runtime + aslist(r["engineCommand"]),
shell=False,
close_fds=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(stdoutdata, stderrdata) = sp.communicate(json.dumps(inp) + "\n\n")
if sp.returncode != 0:
raise WorkflowException(
"Expression engine returned non-zero exit code.")
return json.loads(stdoutdata)
raise WorkflowException("Unknown expression engine '%s'" % ex["engine"])
def exeval(ex, jobinput, requirements, docpath, context, pull_image):
if ex["engine"] == "JsonPointer":
return ref_resolver.resolve_pointer({"job": jobinput, "context": context}, ex["script"])
for r in reversed(requirements):
if r["class"] == "ExpressionEngineRequirement" and r["id"] == ex["engine"]:
if r["id"][0] != "#":
with open(os.path.join(docpath, r["id"])) as f:
ex_obj = yaml.load(f)
sch = process.get_schema()
validate.validate_ex(sch.get_name("ExpressionEngineRequirement", ""), ex_obj)
r = ex_obj
runtime = []
img_id = docker.get_from_requirements(r.get("requirements"), r.get("hints"), pull_image)
if img_id:
runtime = ["docker", "run", "-i", "--rm", img_id]
exdefs = []
for exdef in r.get("expressionDefs", []):
if isinstance(exdef, dict) and "ref" in exdef:
with open(os.path.join(r["_docpath"], exdef["ref"])) as f:
exdefs.append(f.read())
elif isinstance(exdef, basestring):
exdefs.append(exdef)
inp = {
"script": ex["script"],
"expressionDefs": exdefs,
"job": jobinput,
"context": context
}
_logger.debug(json.dumps(inp))
sp = subprocess.Popen(runtime + aslist(r["engineCommand"]),
shell=False,
close_fds=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(stdoutdata, stderrdata) = sp.communicate(json.dumps(inp) + "\n\n")
if sp.returncode != 0:
raise WorkflowException("Expression engine returned non-zero exit code.")
return json.loads(stdoutdata)
raise WorkflowException("Unknown expression engine '%s'" % ex["engine"])
def load_tool(argsworkflow, updateonly, strict, makeTool, debug, print_pre=False):
(document_loader, avsc_names, schema_metadata) = process.get_schema()
uri = "file://" + os.path.abspath(argsworkflow)
fileuri, urifrag = urlparse.urldefrag(uri)
workflowobj = document_loader.fetch(fileuri)
if isinstance(workflowobj, list):
workflowobj = {"cwlVersion": "https://w3id.org/cwl/cwl#draft-2",
"id": fileuri,
"@graph": workflowobj}
workflowobj = update.update(workflowobj, document_loader, fileuri)
document_loader.idx.clear()
if updateonly:
print json.dumps(workflowobj, indent=4)
return 0
try:
processobj, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, workflowobj, strict)
except (schema_salad.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if debug else False))
return 1
if print_pre:
print json.dumps(processobj, indent=4)
return 0
if urifrag:
processobj, _ = document_loader.resolve_ref(uri)
elif isinstance(processobj, list):
processobj, _ = document_loader.resolve_ref(urlparse.urljoin(argsworkflow, "#main"))
try:
t = makeTool(processobj, strict=strict, makeTool=makeTool)
except (schema_salad.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if debug else False))
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error("Tool definition failed initialization:\n%s", e, exc_info=(e if debug else False))
return 1
return t
def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, parser=None):
if args is None:
args = sys.argv[1:]
if parser is None:
parser = arg_parser()
args = parser.parse_args(args)
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
_logger.setLevel(logging.DEBUG)
pkg = pkg_resources.require("cwltool")
if pkg:
if args.version:
print "%s %s" % (sys.argv[0], pkg[0].version)
return 0
else:
_logger.info("%s %s", sys.argv[0], pkg[0].version)
(j, names) = process.get_schema()
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
loader = create_loader(ctx)
if args.print_jsonld_context:
j = {"@context": ctx}
print json.dumps(j, indent=4, sort_keys=True)
return 0
if args.print_rdfs:
print(g.serialize(format=args.rdf_serializer))
return 0
if args.print_spec:
avro_ld.makedoc.avrold_doc(j, sys.stdout)
return 0
if args.print_avro:
print "["
print ", ".join([json.dumps(names.names[n].to_json(), indent=4, sort_keys=True) for n in names.names])
print "]"
return 0
if not args.workflow:
parser.print_help()
_logger.error("")
_logger.error("CWL document required")
return 1
idx = {}
try:
processobj = loader.resolve_ref(args.workflow)
except (avro_ld.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if args.debug else False))
return 1
if args.print_pre:
print json.dumps(processobj, indent=4)
return 0
try:
loader.validate_links(processobj)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if args.debug else False))
return 1
if isinstance(processobj, list):
processobj = loader.resolve_ref(urlparse.urljoin(args.workflow, "#main"))
try:
t = makeTool(processobj, strict=args.strict, makeTool=makeTool)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if args.debug else False))
if args.debug:
_logger.exception("")
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error("Tool definition failed initialization:\n%s", e, exc_info=(e if args.debug else False))
if args.debug:
_logger.exception()
return 1
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.print_dot:
printdot(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.tmp_outdir_prefix != 'tmp':
# Use user defined temp directory (if it exists)
args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix)
if not os.path.exists(args.tmp_outdir_prefix):
_logger.error("Intermediate output directory prefix doesn't exist, reverting to default")
return 1
if args.tmpdir_prefix != 'tmp':
# Use user defined prefix (if the folder exists)
args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix)
if not os.path.exists(args.tmpdir_prefix):
_logger.error("Temporary directory prefix doesn't exist.")
return 1
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
else:
job_order_file = None
if job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(e)
return 1
else:
job_order_object = {}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s", job_order_object)
else:
job_order_object = None
if not job_order_object:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
try:
out = executor(t, job_order_object,
input_basedir, args,
conformance_test=args.conformance_test,
dry_run=args.dry_run,
outdir=args.outdir,
tmp_outdir_prefix=args.tmp_outdir_prefix,
use_container=args.use_container,
pull_image=args.enable_pull,
rm_container=args.rm_container,
tmpdir_prefix=args.tmpdir_prefix,
rm_tmpdir=args.rm_tmpdir,
makeTool=makeTool,
move_outputs=args.move_outputs
)
# This is the workflow output, it needs to be written
sys.stdout.write(json.dumps(out, indent=4))
except (validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s", e, exc_info=(e if args.debug else False))
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n %s", e, exc_info=(e if args.debug else False))
return 1
return 0
def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, parser=None):
if args is None:
args = sys.argv[1:]
if parser is None:
parser = arg_parser()
args = parser.parse_args(args)
if args.verbose:
logging.getLogger("cwltool").setLevel(logging.INFO)
if args.debug:
logging.getLogger("cwltool").setLevel(logging.DEBUG)
(j, names) = process.get_schema()
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
url_fields = []
for c in ctx:
if c != "id" and (ctx[c] == "@id") or (isinstance(ctx[c], dict) and ctx[c].get("@type") == "@id"):
url_fields.append(c)
loader.url_fields = url_fields
loader.idx["cwl:JsonPointer"] = {}
if args.print_jsonld_context:
print json.dumps(ctx, indent=4, sort_keys=True)
return 0
if args.print_rdfs:
print(g.serialize(format=args.rdf_serializer))
return 0
if args.print_spec:
avro_ld.makedoc.avrold_doc(j, sys.stdout)
return 0
if args.print_avro:
print "["
print ", ".join([json.dumps(names.names[n].to_json(), indent=4, sort_keys=True) for n in names.names])
print "]"
return 0
if not args.workflow:
parser.print_help()
_logger.error("")
_logger.error("CWL document required")
return 1
idx = {}
try:
processobj = loader.resolve_ref(args.workflow)
except (avro_ld.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
if args.print_pre:
print json.dumps(processobj, indent=4)
return 0
try:
loader.validate_links(processobj)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
if args.job_order:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(args.job_order))
else:
input_basedir = args.basedir
if isinstance(processobj, list):
processobj = loader.resolve_ref(urlparse.urljoin(args.workflow, "#main"))
try:
t = makeTool(processobj, strict=args.strict, makeTool=makeTool)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error(e)
if args.debug:
_logger.exception()
return 1
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.print_dot:
printdot(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if not args.job_order:
parser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
try:
out = executor(t, loader.resolve_ref(args.job_order),
input_basedir, args,
conformance_test=args.conformance_test,
dry_run=args.dry_run,
outdir=args.outdir,
use_container=args.use_container,
pull_image=args.enable_pull,
rm_container=args.rm_container,
rm_tmpdir=args.rm_tmpdir,
makeTool=makeTool,
move_outputs=args.move_outputs)
print json.dumps(out, indent=4)
except (validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n %s" % e)
if args.debug:
_logger.exception("")
return 1
return 0
def load_tool(argsworkflow,
updateonly,
strict,
makeTool,
debug,
print_pre=False,
print_rdf=False,
print_dot=False,
print_deps=False,
relative_deps=False,
rdf_serializer=None,
stdout=sys.stdout,
urifrag=None):
(document_loader, avsc_names, schema_metadata) = process.get_schema()
if isinstance(avsc_names, Exception):
raise avsc_names
jobobj = None
if isinstance(argsworkflow, basestring):
split = urlparse.urlsplit(argsworkflow)
if split.scheme:
uri = argsworkflow
else:
uri = "file://" + os.path.abspath(argsworkflow)
fileuri, urifrag = urlparse.urldefrag(uri)
workflowobj = document_loader.fetch(fileuri)
if isinstance(workflowobj, list):
# bare list without a version must be treated as draft-2
workflowobj = {
"cwlVersion": "https://w3id.org/cwl/cwl#draft-2",
"id": fileuri,
"@graph": workflowobj
}
elif isinstance(argsworkflow, dict):
workflowobj = argsworkflow
uri = urifrag
fileuri = ""
else:
raise schema_salad.validate.ValidationException("Must be URI or dict")
if "cwl:tool" in workflowobj:
jobobj = workflowobj
workflowobj = document_loader.fetch(
urlparse.urljoin(uri, workflowobj["cwl:tool"]))
workflowobj = update.update(workflowobj, document_loader, fileuri)
document_loader.idx.clear()
if updateonly:
stdout.write(json.dumps(workflowobj, indent=4))
return 0
if print_deps:
printdeps(workflowobj, document_loader, stdout, relative_deps)
return 0
try:
processobj, metadata = schema_salad.schema.load_and_validate(
document_loader, avsc_names, workflowobj, strict)
except (schema_salad.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s",
e,
exc_info=(e if debug else False))
return 1
if print_pre:
stdout.write(json.dumps(processobj, indent=4))
return 0
if print_rdf:
printrdf(argsworkflow, processobj, document_loader.ctx, rdf_serializer,
stdout)
return 0
if print_dot:
printdot(argsworkflow, processobj, document_loader.ctx, stdout)
return 0
if urifrag:
processobj, _ = document_loader.resolve_ref(uri)
elif isinstance(processobj, list):
if 1 == len(processobj):
processobj = processobj[0]
else:
_logger.error(
"Tool file contains graph of multiple objects, must specify one of #%s",
", #".join(
urlparse.urldefrag(i["id"])[1] for i in processobj
if "id" in i))
return 1
try:
t = makeTool(processobj,
strict=strict,
makeTool=makeTool,
loader=document_loader,
avsc_names=avsc_names)
except (schema_salad.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s",
e,
exc_info=(e if debug else False))
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error("Tool definition failed initialization:\n%s",
e,
exc_info=(e if debug else False))
return 1
if jobobj:
for inp in t.tool["inputs"]:
if shortname(inp["id"]) in jobobj:
inp["default"] = jobobj[shortname(inp["id"])]
if metadata:
t.metadata = metadata
else:
t.metadata = {
"$namespaces": t.tool.get("$namespaces", {}),
"$schemas": t.tool.get("$schemas", [])
}
return t
def load_tool(argsworkflow, updateonly, strict, makeTool, debug,
print_pre=False,
print_rdf=False,
print_dot=False,
print_deps=False,
relative_deps=False,
rdf_serializer=None,
stdout=sys.stdout,
urifrag=None):
(document_loader, avsc_names, schema_metadata) = process.get_schema()
if isinstance(avsc_names, Exception):
raise avsc_names
jobobj = None
if isinstance(argsworkflow, basestring):
split = urlparse.urlsplit(argsworkflow)
if split.scheme:
uri = argsworkflow
else:
uri = "file://" + os.path.abspath(argsworkflow)
fileuri, urifrag = urlparse.urldefrag(uri)
workflowobj = document_loader.fetch(fileuri)
elif isinstance(argsworkflow, dict):
workflowobj = argsworkflow
uri = urifrag
fileuri = "#"
else:
raise schema_salad.validate.ValidationException("Must be URI or dict")
if "cwl:tool" in workflowobj:
jobobj = workflowobj
uri = urlparse.urljoin(uri, jobobj["cwl:tool"])
fileuri, urifrag = urlparse.urldefrag(uri)
workflowobj = document_loader.fetch(fileuri)
del jobobj["cwl:tool"]
if isinstance(workflowobj, list):
# bare list without a version must be treated as draft-2
workflowobj = {"cwlVersion": "https://w3id.org/cwl/cwl#draft-2",
"id": fileuri,
"@graph": workflowobj}
workflowobj = update.update(workflowobj, document_loader, fileuri)
document_loader.idx.clear()
if updateonly:
stdout.write(json.dumps(workflowobj, indent=4))
return 0
if print_deps:
printdeps(workflowobj, document_loader, stdout, relative_deps)
return 0
try:
processobj, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, workflowobj, strict)
except (schema_salad.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if debug else False))
return 1
if print_pre:
stdout.write(json.dumps(processobj, indent=4))
return 0
if print_rdf:
printrdf(argsworkflow, processobj, document_loader.ctx, rdf_serializer, stdout)
return 0
if print_dot:
printdot(argsworkflow, processobj, document_loader.ctx, stdout)
return 0
if urifrag:
processobj, _ = document_loader.resolve_ref(uri)
elif isinstance(processobj, list):
if 1 == len(processobj):
processobj = processobj[0]
else:
_logger.error("Tool file contains graph of multiple objects, must specify one of #%s",
", #".join(urlparse.urldefrag(i["id"])[1]
for i in processobj if "id" in i))
return 1
try:
t = makeTool(processobj, strict=strict, makeTool=makeTool, loader=document_loader, avsc_names=avsc_names)
except (schema_salad.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s", e, exc_info=(e if debug else False))
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error("Tool definition failed initialization:\n%s", e, exc_info=(e if debug else False))
return 1
if jobobj:
for inp in t.tool["inputs"]:
if shortname(inp["id"]) in jobobj:
inp["default"] = jobobj[shortname(inp["id"])]
if metadata:
t.metadata = metadata
else:
t.metadata = {"$namespaces": t.tool.get("$namespaces", {}), "$schemas": t.tool.get("$schemas", [])}
return t
def main(args=None,
executor=single_job_executor,
makeTool=workflow.defaultMakeTool,
parser=None):
if args is None:
args = sys.argv[1:]
if parser is None:
parser = arg_parser()
args = parser.parse_args(args)
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
_logger.setLevel(logging.DEBUG)
pkg = pkg_resources.require("cwltool")
if pkg:
if args.version:
print "%s %s" % (sys.argv[0], pkg[0].version)
return 0
else:
_logger.info("%s %s", sys.argv[0], pkg[0].version)
(j, names) = process.get_schema()
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
loader = create_loader(ctx)
if args.print_jsonld_context:
j = {"@context": ctx}
print json.dumps(j, indent=4, sort_keys=True)
return 0
if args.print_rdfs:
print(g.serialize(format=args.rdf_serializer))
return 0
if args.print_spec:
avro_ld.makedoc.avrold_doc(j, sys.stdout)
return 0
if args.print_avro:
print "["
print ", ".join([
json.dumps(names.names[n].to_json(), indent=4, sort_keys=True)
for n in names.names
])
print "]"
return 0
if not args.workflow:
parser.print_help()
_logger.error("")
_logger.error("CWL document required")
return 1
idx = {}
try:
processobj = loader.resolve_ref(args.workflow)
except (avro_ld.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s",
e,
exc_info=(e if args.debug else False))
return 1
if args.print_pre:
print json.dumps(processobj, indent=4)
return 0
try:
loader.validate_links(processobj)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s",
e,
exc_info=(e if args.debug else False))
return 1
if isinstance(processobj, list):
processobj = loader.resolve_ref(
urlparse.urljoin(args.workflow, "#main"))
try:
t = makeTool(processobj, strict=args.strict, makeTool=makeTool)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s",
e,
exc_info=(e if args.debug else False))
if args.debug:
_logger.exception("")
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error("Tool definition failed initialization:\n%s",
e,
exc_info=(e if args.debug else False))
if args.debug:
_logger.exception()
return 1
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.print_dot:
printdot(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.tmp_outdir_prefix != 'tmp':
# Use user defined temp directory (if it exists)
args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix)
if not os.path.exists(args.tmp_outdir_prefix):
_logger.error(
"Intermediate output directory prefix doesn't exist, reverting to default"
)
return 1
if args.tmpdir_prefix != 'tmp':
# Use user defined prefix (if the folder exists)
args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix)
if not os.path.exists(args.tmpdir_prefix):
_logger.error("Temporary directory prefix doesn't exist.")
return 1
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
else:
job_order_file = None
if job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(e)
return 1
else:
job_order_object = {}
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s",
job_order_object)
else:
job_order_object = None
if not job_order_object:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
try:
out = executor(t,
job_order_object,
input_basedir,
args,
conformance_test=args.conformance_test,
dry_run=args.dry_run,
outdir=args.outdir,
tmp_outdir_prefix=args.tmp_outdir_prefix,
use_container=args.use_container,
pull_image=args.enable_pull,
rm_container=args.rm_container,
tmpdir_prefix=args.tmpdir_prefix,
rm_tmpdir=args.rm_tmpdir,
makeTool=makeTool,
move_outputs=args.move_outputs)
# This is the workflow output, it needs to be written
sys.stdout.write(json.dumps(out, indent=4))
except (validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s",
e,
exc_info=(e if args.debug else False))
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n %s",
e,
exc_info=(e if args.debug else False))
return 1
return 0
def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, parser=None):
if args is None:
args = sys.argv[1:]
if parser is None:
parser = arg_parser()
args = parser.parse_args(args)
if args.verbose:
logging.getLogger("cwltool").setLevel(logging.INFO)
if args.debug:
logging.getLogger("cwltool").setLevel(logging.DEBUG)
(j, names) = process.get_schema()
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
url_fields = []
for c in ctx:
if c != "id" and (ctx[c] == "@id") or (isinstance(ctx[c], dict) and ctx[c].get("@type") == "@id"):
url_fields.append(c)
loader.url_fields = url_fields
loader.idx["cwl:JsonPointer"] = {}
if args.print_jsonld_context:
j = {"@context": ctx}
print json.dumps(j, indent=4, sort_keys=True)
return 0
if args.print_rdfs:
print(g.serialize(format=args.rdf_serializer))
return 0
if args.print_spec:
avro_ld.makedoc.avrold_doc(j, sys.stdout)
return 0
if args.print_avro:
print "["
print ", ".join([json.dumps(names.names[n].to_json(), indent=4, sort_keys=True) for n in names.names])
print "]"
return 0
if not args.workflow:
parser.print_help()
_logger.error("")
_logger.error("CWL document required")
return 1
idx = {}
try:
processobj = loader.resolve_ref(args.workflow)
except (avro_ld.validate.ValidationException, RuntimeError) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
if args.print_pre:
print json.dumps(processobj, indent=4)
return 0
try:
loader.validate_links(processobj)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
if args.job_order:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(args.job_order))
else:
input_basedir = args.basedir
if isinstance(processobj, list):
processobj = loader.resolve_ref(urlparse.urljoin(args.workflow, "#main"))
try:
t = makeTool(processobj, strict=args.strict, makeTool=makeTool)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
except (RuntimeError, workflow.WorkflowException) as e:
_logger.error(e)
if args.debug:
_logger.exception()
return 1
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.print_dot:
printdot(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if not args.job_order:
parser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
out = executor(t, loader.resolve_ref(args.job_order),
input_basedir, args,
conformance_test=args.conformance_test,
dry_run=args.dry_run,
outdir=args.outdir,
use_container=args.use_container,
pull_image=args.enable_pull,
rm_container=args.rm_container,
tmpdir_prefix=args.tmpdir_prefix,
rm_tmpdir=args.rm_tmpdir,
makeTool=makeTool,
move_outputs=args.move_outputs
)
try:
pass
except (validate.ValidationException) as e:
print json.dumps(out, indent=4)
_logger.error("Input object failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n %s" % e)
if args.debug:
_logger.exception("")
return 1
return 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument("workflow", type=str, nargs="?", default=None)
parser.add_argument("job_order", type=str, nargs="?", default=None)
parser.add_argument("--conformance-test", action="store_true")
parser.add_argument("--basedir", type=str)
parser.add_argument("--outdir", type=str)
parser.add_argument("--no-container", action="store_true", help="Do not execute jobs in a Docker container, even when specified by the CommandLineTool")
parser.add_argument("--leave-container", action="store_true", help="Do not delete Docker container used by jobs after they exit")
parser.add_argument("--no-pull", default=False, action="store_true", help="Do not try to pull Docker images")
parser.add_argument("--dry-run", action="store_true", help="Load and validate but do not execute")
parser.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for workflow")
parser.add_argument("--rdf-serializer", help="Output RDF serialization format (one of turtle (default), n3, nt, xml)", default="turtle")
parser.add_argument("--print-spec", action="store_true", help="Print HTML specification document")
parser.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for CWL file")
parser.add_argument("--print-rdfs", action="store_true", help="Print JSON-LD context for CWL file")
parser.add_argument("--print-avro", action="store_true", help="Print Avro schema")
parser.add_argument("--print-pre", action="store_true", help="Print workflow document after preprocessing")
parser.add_argument("--strict", action="store_true", help="Strict validation (unrecognized fields are an error) (default false)")
parser.add_argument("--verbose", action="store_true", help="Print more logging")
parser.add_argument("--debug", action="store_true", help="Print even more logging")
args = parser.parse_args()
if args.verbose:
logging.getLogger("cwltool").setLevel(logging.INFO)
if args.debug:
logging.getLogger("cwltool").setLevel(logging.DEBUG)
(j, names) = process.get_schema()
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
url_fields = []
for c in ctx:
if c != "id" and (ctx[c] == "@id") or (isinstance(ctx[c], dict) and ctx[c].get("@type") == "@id"):
url_fields.append(c)
loader.url_fields = url_fields
loader.idx["cwl:JsonPointer"] = {}
if args.print_jsonld_context:
print json.dumps(ctx, indent=4, sort_keys=True)
return 0
if args.print_rdfs:
print(g.serialize(format=args.rdf_serializer))
return 0
if args.print_spec:
avro_ld.makedoc.avrold_doc(j, sys.stdout)
return 0
if args.print_avro:
print "["
print ", ".join([json.dumps(names.names[n].to_json(), indent=4, sort_keys=True) for n in names.names])
print "]"
return 0
if not args.workflow:
_logger.error("CWL document required")
parser.print_help()
return 1
idx = {}
try:
processobj = loader.resolve_ref(args.workflow)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception("")
return 1
#_logger.warn(url_fields)
#_logger.warn(json.dumps(loader.idx, indent=4))
if args.print_pre:
print json.dumps(processobj, indent=4)
return 0
try:
loader.validate_links(processobj)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
if args.job_order:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(args.job_order))
else:
input_basedir = args.basedir
if isinstance(processobj, list):
processobj = loader.resolve_ref(urlparse.urljoin(args.workflow, "#main"))
try:
t = workflow.makeTool(processobj, input_basedir, strict=args.strict)
except (avro_ld.validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
except RuntimeError as e:
_logger.error(e)
if args.debug:
_logger.exception()
return 1
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if not args.job_order:
_logger.error("Input object required")
_logger.error("Use --help for command line options")
return 1
try:
final_output = []
def output_callback(out, processStatus):
if processStatus == "success":
_logger.info("Overall job status is %s", processStatus)
else:
_logger.warn("Overall job status is %s", processStatus)
final_output.append(out)
jobiter = t.job(loader.resolve_ref(args.job_order), input_basedir, output_callback, use_container=(not args.no_container))
if args.conformance_test:
job = jobiter.next()
a = {"args": job.command_line}
if job.stdin:
a["stdin"] = job.stdin
if job.stdout:
a["stdout"] = job.stdout
if job.generatefiles:
a["generatefiles"] = job.generatefiles
print json.dumps(a)
else:
last = None
for r in jobiter:
if r:
if args.dry_run:
outdir = "/tmp"
elif args.outdir:
outdir = args.outdir
else:
outdir = tempfile.mkdtemp()
r.run(outdir, dry_run=args.dry_run, pull_image=(not args.no_pull), rm_container=(not args.leave_container))
else:
print "Workflow deadlocked."
return 1
last = r
_logger.info("Output directory is %s", outdir)
print json.dumps(final_output[0], indent=4)
except (validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n%s" % e)
if args.debug:
_logger.exception()
return 1
return 0
