def find_files(adapter, job):
if "value" in adapter:
value = resolve_eval(job, adapter["value"])
else:
return None
schema = adapter["schema"]
if "$ref" in schema:
schema = from_url(schema["$ref"], adapter.get("$ref_base_url"))
if 'oneOf' in schema:
for a in schema['oneOf']:
v = find_files(a, value)
if v is not None:
return v
return None
elif 'type' in schema:
if schema["type"] == "array" and isinstance(value, list):
return [find_files({"value": v,
"schema": schema["items"]}, job) for v in value]
elif schema["type"] == "object" and isinstance(value, dict):
if "path" in value:
return value["path"]
else:
raise Exception("Not expecting a dict %s" % (value))
return None
def find_files(self, adapter, job):
if "value" in adapter:
value = self.resolve_eval(job, adapter["value"])
else:
return None
schema = adapter["schema"]
if "$ref" in schema:
schema = from_url(schema["$ref"], self.ref_base_url)
if 'oneOf' in schema:
for a in schema['oneOf']:
v = self.find_files(a, value)
if v is not None:
return v
return None
elif 'type' in schema:
if schema["type"] == "array" and isinstance(value, list):
return [
self.find_files({
"value": v,
"schema": schema["items"]
}, job) for v in value
]
elif schema["type"] == "object" and isinstance(value, dict):
if "path" in value:
return value["path"]
else:
raise Exception("Not expecting a dict %s" % (value))
return None
def to_str(schema, value, base_url, path_mapper):
if "$ref" in schema:
schema = from_url(schema["$ref"], base_url)
if 'oneOf' in schema:
for a in schema['oneOf']:
v = to_str(a, value, base_url, path_mapper)
if v is not None:
return v
return None
elif 'type' in schema:
if schema["type"] == "array" and isinstance(value, list):
return [to_str(schema["items"], v, base_url, path_mapper) for v in value]
elif schema["type"] == "object" and isinstance(value, dict):
if "path" in value:
return path_mapper(value["path"])
else:
raise Exception("Not expecting a dict %s" % (value))
elif schema["type"] in ("string", "number", "integer"):
return str(value)
elif schema["type"] == "boolean":
# need special handling for flags
return str(value)
return None
def to_str(self, schema, value, path_mapper):
if "$ref" in schema:
schema = from_url(schema["$ref"], self.ref_base_url)
if 'oneOf' in schema:
for a in schema['oneOf']:
v = self.to_str(a, value, path_mapper)
if v is not None:
return v
return None
elif 'type' in schema:
if schema["type"] == "array" and isinstance(value, list):
return [
self.to_str(schema["items"], v, path_mapper) for v in value
]
elif schema["type"] == "object" and isinstance(value, dict):
if "path" in value:
return path_mapper(value["path"])
else:
raise Exception("Not expecting a dict %s" % (value))
elif schema["type"] in ("string", "number", "integer"):
return str(value)
elif schema["type"] == "boolean":
# handled specially by adapt()
return value
return None
def main():
parser = argparse.ArgumentParser()
parser.add_argument("tool", type=str)
parser.add_argument("job_order", type=str)
parser.add_argument("--conformance-test", action="store_true")
parser.add_argument("--basedir", type=str)
parser.add_argument("--no-container", action="store_true")
parser.add_argument("-x", action="store_true", help="Execute")
args = parser.parse_args()
try:
t = tool.Tool(from_url(args.tool))
except jsonschema.exceptions.ValidationError as e:
print "Tool definition failed validation"
print e
return 1
basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(args.job_order))
try:
job = t.job(from_url(args.job_order), basedir, use_container=(not args.no_container))
if args.conformance_test:
a = {"args": job.command_line}
if job.stdin:
a["stdin"] = job.stdin
if job.stdout:
a["stdout"] = job.stdout
print json.dumps(a)
else:
print '%s%s%s' % (' '.join(job.command_line),
' < %s' % (job.stdin) if job.stdin else '',
' > %s' % (job.stdout) if job.stdout else '')
except jsonschema.exceptions.ValidationError as e:
print "Job order failed validation"
print e
return 1
if args.x:
job.run()
return 0
def __init__(self, toolpath_object, docpath):
self.impl = toolpath_object["impl"]
try:
self.embedded_tool = makeTool(
from_url(os.path.join(docpath, self.impl)), docpath)
except validate.ValidationException as v:
raise WorkflowException(
"Tool definition %s failed validation:\n%s" %
(os.path.join(docpath, self.impl), validate.indent(str(v))))
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step_" + str(random.randint(1, 1000000000))
for i in toolpath_object["inputs"]:
d = i["def"][len(self.impl):]
toolid = i.get("id", self.id + "." + idk(d))
found = False
for a in self.embedded_tool.tool["inputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException(
"Did not find input '%s' in external process" % (i["def"]))
i["id"] = toolid
for i in toolpath_object["outputs"]:
d = i["def"][len(self.impl):]
toolid = i["id"]
found = False
for a in self.embedded_tool.tool["outputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException(
"Did not find output '%s' in external process" %
(i["def"]))
i["id"] = toolid
super(External, self).__init__(toolpath_object, "Process", docpath)
def __init__(self, toolpath_object, docpath):
self.impl = toolpath_object["impl"]
try:
self.embedded_tool = makeTool(from_url(os.path.join(docpath, self.impl)), docpath)
except validate.ValidationException as v:
raise WorkflowException("Tool definition %s failed validation:\n%s" % (os.path.join(docpath, self.impl), validate.indent(str(v))))
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step_" + str(random.randint(1, 1000000000))
for i in toolpath_object["inputs"]:
d = i["def"][len(self.impl):]
toolid = i.get("id", self.id + "." + idk(d))
found = False
for a in self.embedded_tool.tool["inputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException("Did not find input '%s' in external process" % (i["def"]))
i["id"] = toolid
for i in toolpath_object["outputs"]:
d = i["def"][len(self.impl):]
toolid = i["id"]
found = False
for a in self.embedded_tool.tool["outputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException("Did not find output '%s' in external process" % (i["def"]))
i["id"] = toolid
super(External, self).__init__(toolpath_object, "External", docpath)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("workflow", type=str)
parser.add_argument("job_order", type=str, nargs="?", default=None)
parser.add_argument("--conformance-test", action="store_true")
parser.add_argument("--basedir", type=str)
parser.add_argument("--outdir", type=str)
parser.add_argument("--no-container", action="store_true", help="Do not execute jobs in a Docker container, even when specified by the CommandLineTool")
parser.add_argument("--leave-container", action="store_true", help="Do not delete Docker container after it exits")
parser.add_argument("--no-pull", default=False, action="store_true", help="Do not try to pull the Docker image")
parser.add_argument("--dry-run", action="store_true", help="Do not execute")
parser.add_argument("--verbose", action="store_true", help="Print more logging")
parser.add_argument("--debug", action="store_true", help="Print even more logging")
parser.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for workflow")
parser.add_argument("--rdf-serializer", help="Output RDF serialization format (one of turtle (default), n3, nt, xml)", default="turtle")
args = parser.parse_args()
if args.verbose:
logging.getLogger("cwltool").setLevel(logging.INFO)
if args.debug:
logging.getLogger("cwltool").setLevel(logging.DEBUG)
if args.print_rdf:
printrdf(args.workflow, args.rdf_serializer)
return 0
if not args.job_order:
_logger.error("Input object required")
return 1
basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(args.job_order))
try:
t = workflow.makeTool(from_url(args.workflow), basedir)
except (jsonschema.exceptions.ValidationError, validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
except RuntimeError as e:
_logger.error(e)
if args.debug:
_logger.exception()
return 1
try:
final_output = []
def output_callback(out):
final_output.append(out)
jobiter = t.job(from_url(args.job_order), basedir, output_callback, use_container=(not args.no_container))
if args.conformance_test:
job = jobiter.next()
a = {"args": job.command_line}
if job.stdin:
a["stdin"] = job.stdin
if job.stdout:
a["stdout"] = job.stdout
if job.generatefiles:
a["generatefiles"] = job.generatefiles
print json.dumps(a)
else:
last = None
for r in jobiter:
if r:
if args.dry_run:
outdir = "/tmp"
elif args.outdir:
outdir = args.outdir
else:
outdir = tempfile.mkdtemp()
r.run(outdir, dry_run=args.dry_run, pull_image=(not args.no_pull), rm_container=(not args.leave_container))
else:
print "Workflow deadlocked."
return 1
last = r
_logger.info("Output directory is %s", outdir)
print json.dumps(final_output[0])
except (jsonschema.exceptions.ValidationError, validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n%s" % e)
if args.debug:
_logger.exception()
return 1
return 0
def printrdf(workflow, sr):
from rdflib import Graph, plugin
from rdflib.serializer import Serializer
wf = from_url(workflow)
g = Graph().parse(data=json.dumps(wf), format='json-ld', location=workflow)
print(g.serialize(format=sr))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("workflow", type=str, nargs="?", default=None)
parser.add_argument("job_order", type=str, nargs="?", default=None)
parser.add_argument("--conformance-test", action="store_true")
parser.add_argument("--basedir", type=str)
parser.add_argument("--outdir", type=str)
parser.add_argument("--no-container", action="store_true", help="Do not execute jobs in a Docker container, even when specified by the CommandLineTool")
parser.add_argument("--leave-container", action="store_true", help="Do not delete Docker container used by jobs after they exit")
parser.add_argument("--no-pull", default=False, action="store_true", help="Do not try to pull Docker images")
parser.add_argument("--dry-run", action="store_true", help="Load and validate but do not execute")
parser.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for workflow")
parser.add_argument("--rdf-serializer", help="Output RDF serialization format (one of turtle (default), n3, nt, xml)", default="turtle")
parser.add_argument("--print-spec", action="store_true", help="Print HTML specification document")
parser.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for CWL file")
parser.add_argument("--print-rdfs", action="store_true", help="Print JSON-LD context for CWL file")
parser.add_argument("--verbose", action="store_true", help="Print more logging")
parser.add_argument("--debug", action="store_true", help="Print even more logging")
args = parser.parse_args()
if args.verbose:
logging.getLogger("cwltool").setLevel(logging.INFO)
if args.debug:
logging.getLogger("cwltool").setLevel(logging.DEBUG)
cwl_avsc = os.path.join(module_dir, 'schemas/draft-2/cwl-avro.yml')
if args.print_jsonld_context:
with open(cwl_avsc) as f:
j = yaml.load(f)
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
print json.dumps(ctx, indent=4, sort_keys=True)
return 0
if args.print_rdfs:
with open(cwl_avsc) as f:
j = yaml.load(f)
(ctx, g) = avro_ld.jsonld_context.avrold_to_jsonld_context(j)
print(g.serialize(format=args.rdf_serializer))
return 0
if args.print_spec:
with open(cwl_avsc) as f:
j = yaml.load(f)
avro_ld.makedoc.avrold_doc(j, sys.stdout)
return 0
if not args.workflow:
_logger.error("CWL document required")
parser.print_help()
return 1
if args.print_rdf:
printrdf(args.workflow, args.rdf_serializer)
return 0
if not args.job_order:
_logger.error("Input object required")
parser.print_help()
return 1
basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(args.job_order))
try:
t = workflow.makeTool(from_url(args.workflow), basedir)
except (jsonschema.exceptions.ValidationError, validate.ValidationException) as e:
_logger.error("Tool definition failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
except RuntimeError as e:
_logger.error(e)
if args.debug:
_logger.exception()
return 1
try:
final_output = []
def output_callback(out):
final_output.append(out)
jobiter = t.job(from_url(args.job_order), basedir, output_callback, use_container=(not args.no_container))
if args.conformance_test:
job = jobiter.next()
a = {"args": job.command_line}
if job.stdin:
a["stdin"] = job.stdin
if job.stdout:
a["stdout"] = job.stdout
if job.generatefiles:
a["generatefiles"] = job.generatefiles
print json.dumps(a)
else:
last = None
for r in jobiter:
if r:
if args.dry_run:
outdir = "/tmp"
elif args.outdir:
outdir = args.outdir
else:
outdir = tempfile.mkdtemp()
r.run(outdir, dry_run=args.dry_run, pull_image=(not args.no_pull), rm_container=(not args.leave_container))
else:
print "Workflow deadlocked."
return 1
last = r
_logger.info("Output directory is %s", outdir)
print json.dumps(final_output[0])
except (jsonschema.exceptions.ValidationError, validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s" % e)
if args.debug:
_logger.exception()
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n%s" % e)
if args.debug:
_logger.exception()
return 1
return 0
