def load_job_order(args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
):
# type: (...) -> Tuple[Dict[Text, Any], Text, Loader]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object:
overrides.extend(resolve_overrides(job_order_object, file_uri(job_order_file), tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if not job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
return (job_order_object, input_basedir, loader)
def test_attachments():
furi = file_uri(get_data("schema_salad/tests/multidoc.yml"))
l1 = Loader({})
r1, _ = l1.resolve_ref(furi)
with open(get_data("schema_salad/tests/multidoc.yml"), "rt") as f:
content = f.read()
assert {"foo": "bar", "baz": content, "quux": content} == r1
l2 = Loader({}, allow_attachments=lambda x: x["foo"] == "bar")
r2, _ = l2.resolve_ref(furi)
assert {
"foo": "bar",
"baz": "This is the {first attachment}.\n",
"quux": "This is the [second attachment].",
} == r2
l3 = Loader({}, allow_attachments=lambda x: x["foo"] == "baz")
r3, _ = l3.resolve_ref(furi)
with open(get_data("schema_salad/tests/multidoc.yml"), "rt") as f:
content = f.read()
assert {"foo": "bar", "baz": content, "quux": content} == r3
def test_fetch_inject_id() -> None:
path = get_data("schema_salad/tests/inject-id1.yml")
assert path
if is_fs_case_sensitive(os.path.dirname(path)):
def lower(item: str) -> str:
return item
else:
def lower(item: str) -> str:
return item.lower()
l1 = Loader({"id": "@id"})
furi1 = file_uri(path)
r1, _ = l1.resolve_ref(furi1)
assert {"id": furi1 + "#foo", "bar": "baz"} == r1
assert [lower(furi1), lower(furi1 + "#foo")] == sorted(
list(lower(k) for k in l1.idx.keys())
)
l2 = Loader({"id": "@id"})
path2 = get_data("schema_salad/tests/inject-id2.yml")
assert path2
furi2 = file_uri(path2)
r2, _ = l2.resolve_ref(furi2)
assert {"id": furi2, "bar": "baz"} == r2
assert [lower(furi2)] == sorted(list(lower(k) for k in l2.idx.keys()))
l3 = Loader({"id": "@id"})
path3 = get_data("schema_salad/tests/inject-id3.yml")
assert path3
furi3 = file_uri(path3)
r3, _ = l3.resolve_ref(furi3)
assert {"id": "http://example.com", "bar": "baz"} == r3
assert [lower(furi3), "http://example.com"] == sorted(
list(lower(k) for k in l3.idx.keys())
)
def load_job_order(
args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides_list, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader]
job_order_object = None
job_order_file = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx,
fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object,
file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object is not None:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file is not None:
input_basedir = args.basedir if args.basedir \
else os.path.abspath(os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object:
ov_uri = file_uri(job_order_file or input_basedir)
overrides_list.extend(
resolve_overrides(job_order_object, ov_uri, tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if job_order_object is None:
input_basedir = args.basedir if args.basedir else os.getcwd()
if job_order_object is not None and not isinstance(job_order_object,
MutableMapping):
_logger.error(
'CWL input object at %s is not formatted correctly, it should be a '
'JSON/YAML dictionay, not %s.\n'
'Raw input object:\n%s', job_order_file or "stdin",
type(job_order_object), job_order_object)
sys.exit(1)
return (job_order_object, input_basedir, loader)
def test_fetch_inject_id():
lower = lambda s: s.lower()
if is_fs_case_sensitive(
os.path.dirname(get_data("schema_salad/tests/inject-id1.yml"))):
lower = lambda a: a
l1 = Loader({"id": "@id"})
furi1 = file_uri(get_data("schema_salad/tests/inject-id1.yml"))
r1, _ = l1.resolve_ref(furi1)
assert {"id": furi1 + "#foo", "bar": "baz"} == r1
assert [lower(furi1), lower(furi1 + "#foo")
] == sorted(list(lower(k) for k in l1.idx.keys()))
l2 = Loader({"id": "@id"})
furi2 = file_uri(get_data("schema_salad/tests/inject-id2.yml"))
r2, _ = l2.resolve_ref(furi2)
assert {"id": furi2, "bar": "baz"} == r2
assert [lower(furi2)] == sorted(list(lower(k) for k in l2.idx.keys()))
l3 = Loader({"id": "@id"})
furi3 = file_uri(get_data("schema_salad/tests/inject-id3.yml"))
r3, _ = l3.resolve_ref(furi3)
assert {"id": "http://example.com", "bar": "baz"} == r3
assert [lower(furi3), "http://example.com"
] == sorted(list(lower(k) for k in l3.idx.keys()))
def test_sourceline() -> None:
ldr = Loader({"id": "@id"})
path = get_data("tests/frag.yml")
assert path
b, _ = ldr.resolve_ref(path)
class TestExp(Exception):
pass
try:
with SourceLine(b, 1, TestExp, False):
raise Exception("Whoops")
except TestExp as e:
assert str(e).endswith("frag.yml:3:3: Whoops"), e
except Exception as exc:
assert False, exc
def test_mixin() -> None:
base_url = file_uri(os.path.join(os.getcwd(), "tests"))
ldr = Loader({})
path = get_data("tests/mixin.yml")
assert path
ra = ldr.resolve_ref(cmap({"$mixin": path, "one": "five"}), base_url=base_url)
assert {"id": "four", "one": "five"} == ra[0]
ldr = Loader({"id": "@id"})
ra = ldr.resolve_all(
cmap([{"id": "a", "m": {"$mixin": path}}, {"id": "b", "m": {"$mixin": path}}]),
base_url=base_url,
)
assert [
{"id": base_url + "#a", "m": {"id": base_url + "#a/four", "one": "two"}},
{"id": base_url + "#b", "m": {"id": base_url + "#b/four", "one": "two"}},
] == ra[0]
def load_job_order(args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides_list, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader]
job_order_object = None
job_order_file = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object is not None:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file is not None:
input_basedir = args.basedir if args.basedir \
else os.path.abspath(os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object:
ov_uri = file_uri(job_order_file or input_basedir)
overrides_list.extend(
resolve_overrides(job_order_object, ov_uri, tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if job_order_object is None:
input_basedir = args.basedir if args.basedir else os.getcwd()
if job_order_object is not None and not isinstance(job_order_object, MutableMapping):
_logger.error(
'CWL input object at %s is not formatted correctly, it should be a '
'JSON/YAML dictionay, not %s.\n'
'Raw input object:\n%s', job_order_file or "stdin",
type(job_order_object), job_order_object)
sys.exit(1)
return (job_order_object, input_basedir, loader)
def load_job_order(
args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
):
# type: (...) -> Tuple[Dict[Text, Any], Text, Loader]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx,
fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object,
file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object:
overrides.extend(
resolve_overrides(job_order_object, file_uri(job_order_file),
tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if not job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
return (job_order_object, input_basedir, loader)
def resolve_and_validate_document(
loadingContext, # type: LoadingContext
workflowobj, # type: Union[CommentedMap, CommentedSeq]
uri, # type: str
preprocess_only=False, # type: bool
skip_schemas=None, # type: Optional[bool]
):
# type: (...) -> Tuple[LoadingContext, str]
"""Validate a CWL document."""
if not loadingContext.loader:
raise ValueError("loadingContext must have a loader.")
else:
loader = loadingContext.loader
loadingContext = loadingContext.copy()
if not isinstance(workflowobj, MutableMapping):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
jobobj, _ = loader.resolve_all(workflowobj, uri)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"]
workflowobj = fetch_document(uri, loadingContext)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
cwlVersion = loadingContext.metadata.get("cwlVersion")
if not cwlVersion:
cwlVersion = workflowobj.get("cwlVersion")
if not cwlVersion and fileuri != uri:
# The tool we're loading is a fragment of a bigger file. Get
# the document root element and look for cwlVersion there.
metadata = fetch_document(fileuri,
loadingContext)[1] # type: Dict[str, Any]
cwlVersion = metadata.get("cwlVersion")
if not cwlVersion:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(cwlVersion, str):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(type(cwlVersion)))
# strip out version
cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion)
if cwlVersion not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
if (isinstance(jobobj, CommentedMap)
and "http://commonwl.org/cwltool#overrides" in jobobj):
loadingContext.overrides_list.extend(
resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
if (isinstance(jobobj, CommentedMap)
and "https://w3id.org/cwl/cwl#requirements" in jobobj):
if cwlVersion not in ("v1.1.0-dev1", "v1.1"):
raise ValidationException(
"`cwl:requirements` in the input object is not part of CWL "
"v1.0. You can adjust to use `cwltool:overrides` instead; or you "
"can set the cwlVersion to v1.1 or greater.")
loadingContext.overrides_list.append({
"overrideTarget":
uri,
"requirements":
jobobj["https://w3id.org/cwl/cwl#requirements"],
})
del jobobj["https://w3id.org/cwl/cwl#requirements"]
(sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, str, None]
document_loader = Loader(
sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=loadingContext.fetcher_constructor,
skip_schemas=skip_schemas,
doc_cache=loadingContext.doc_cache,
)
if cwlVersion == "v1.0":
_add_blank_ids(workflowobj)
document_loader.resolve_all(workflowobj, fileuri)
processobj, metadata = document_loader.resolve_ref(uri)
if loadingContext.metadata:
metadata = loadingContext.metadata
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException(
"Workflow must be a CommentedMap or CommentedSeq.")
if not isinstance(metadata, CommentedMap):
raise ValidationException("metadata must be a CommentedMap, was %s" %
type(metadata))
if isinstance(processobj, CommentedMap):
uri = processobj["id"]
_convert_stdstreams_to_files(workflowobj)
if isinstance(jobobj, CommentedMap):
loadingContext.jobdefaults = jobobj
loadingContext.loader = document_loader
loadingContext.avsc_names = avsc_names
loadingContext.metadata = metadata
if preprocess_only:
return loadingContext, uri
if loadingContext.do_validate:
schema.validate_doc(avsc_names, processobj, document_loader,
loadingContext.strict)
# None means default behavior (do update)
if loadingContext.do_update in (True, None):
if "cwlVersion" not in metadata:
metadata["cwlVersion"] = cwlVersion
processobj = update.update(processobj, document_loader, fileuri,
loadingContext.enable_dev, metadata)
document_loader.idx[processobj["id"]] = processobj
def update_index(pr: CommentedMap) -> None:
if "id" in pr:
document_loader.idx[pr["id"]] = pr
visit_class(processobj,
("CommandLineTool", "Workflow", "ExpressionTool"),
update_index)
return loadingContext, uri
def load_job_order(args,
t,
stdin,
print_input_deps=False,
relative_deps=False,
stdout=sys.stdout,
make_fs_access=None,
fetcher_constructor=None):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[Text, Text], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
_jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(_jobloaderctx,
fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object,
file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[Text, Text]
records = [] # type: List[Text]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap, records)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
for record_name in records:
record = {}
record_items = {
k: v
for k, v in six.iteritems(cmd_line)
if k.startswith(record_name)
}
for key, value in six.iteritems(record_items):
record[key[len(record_name) + 1:]] = value
del cmd_line[key]
cmd_line[str(record_name)] = record
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
else:
job_order_object = {"id": args.workflow}
del cmd_line["job_order"]
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Parsed job order from command line: %s",
json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print(u"\nOptions for {} ".format(args.workflow))
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object,
loader,
stdout,
relative_deps,
"",
basedir=file_uri(input_basedir + "/"))
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
visit_class(job_order_object, ("File", "Directory"), pathToLoc)
adjustDirObjs(job_order_object, trim_listing)
normalizeFilesDirs(job_order_object)
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout):
# type: (argparse.Namespace, Process, argparse.ArgumentParser, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]]
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
"path": {"@type": "@id"},
"format": {"@type": "@id"},
"id": "@id"}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(str(e), exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[str,str]
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(str(e), exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
parser.print_help()
if toolparser:
print u"\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
if print_input_deps:
printdeps(job_order_object, loader, stdout, relative_deps,
basedir=u"file://%s/" % input_basedir)
return 0
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, parser=None):
if args is None:
args = sys.argv[1:]
if parser is None:
parser = arg_parser()
args = parser.parse_args(args)
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
_logger.setLevel(logging.DEBUG)
pkg = pkg_resources.require("cwltool")
if pkg:
if args.version:
print "%s %s" % (sys.argv[0], pkg[0].version)
return 0
else:
_logger.info("%s %s", sys.argv[0], pkg[0].version)
if not args.workflow:
parser.print_help()
_logger.error("")
_logger.error("CWL document required")
return 1
t = load_tool(args.workflow, args.update, args.strict, makeTool, args.debug)
if type(t) == int:
return t
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.print_dot:
printdot(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.tmp_outdir_prefix != 'tmp':
# Use user defined temp directory (if it exists)
args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix)
if not os.path.exists(args.tmp_outdir_prefix):
_logger.error("Intermediate output directory prefix doesn't exist, reverting to default")
return 1
if args.tmpdir_prefix != 'tmp':
# Use user defined prefix (if the folder exists)
args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix)
if not os.path.exists(args.tmpdir_prefix):
_logger.error("Temporary directory prefix doesn't exist.")
return 1
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
else:
job_order_file = None
if args.conformance_test:
loader = Loader({})
else:
loader = Loader({"id": "@id", "path": {"@type": "@id"}})
if job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s", job_order_object)
else:
job_order_object = None
if not job_order_object:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
try:
out = executor(t, job_order_object,
input_basedir, args,
conformance_test=args.conformance_test,
dry_run=args.dry_run,
outdir=args.outdir,
tmp_outdir_prefix=args.tmp_outdir_prefix,
use_container=args.use_container,
preserve_environment=args.preserve_environment,
pull_image=args.enable_pull,
rm_container=args.rm_container,
tmpdir_prefix=args.tmpdir_prefix,
rm_tmpdir=args.rm_tmpdir,
makeTool=makeTool,
move_outputs=args.move_outputs
)
# This is the workflow output, it needs to be written
sys.stdout.write(json.dumps(out, indent=4))
except (validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s", e, exc_info=(e if args.debug else False))
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n %s", e, exc_info=(e if args.debug else False))
return 1
return 0
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
_jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin) # type: ignore
job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[Text, Text]
records = [] # type: List[Text]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap, records)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
for record_name in records:
record = {}
record_items = {
k: v for k, v in cmd_line.iteritems()
if k.startswith(record_name)}
for key, value in record_items.iteritems():
record[key[len(record_name) + 1:]] = value
del cmd_line[key]
cmd_line[str(record_name)] = record
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
else:
job_order_object = {"id": args.workflow}
del cmd_line["job_order"]
job_order_object.update({namemap[k]: v for k, v in cmd_line.items()})
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print(u"\nOptions for {} ".format(args.workflow))
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object, loader, stdout, relative_deps, "",
basedir=file_uri(input_basedir + "/"))
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
visit_class(job_order_object, ("File", "Directory"), pathToLoc)
adjustDirObjs(job_order_object, trim_listing)
normalizeFilesDirs(job_order_object)
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def load_job_order(args,
t,
stdin,
print_input_deps=False,
relative_deps=False,
stdout=sys.stdout):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]]
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
u"path": {
u"@type": u"@id"
},
u"location": {
u"@type": u"@id"
},
u"format": {
u"@type": u"@id"
},
u"id": u"@id"
}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
except Exception as e:
_logger.error(str(e), exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[unicode, unicode]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(str(e),
exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
_logger.debug(u"Parsed job order from command line: %s",
json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print u"\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object,
loader,
stdout,
relative_deps,
"",
basedir=u"file://%s/" % input_basedir)
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
adjustDirObjs(job_order_object, pathToLoc)
adjustFileObjs(job_order_object, pathToLoc)
normalizeFilesDirs(job_order_object)
adjustDirObjs(
job_order_object,
cast(Callable[..., Any],
functools.partial(getListing, StdFsAccess(input_basedir))))
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def load_job_order(args,
t,
parser,
stdin,
print_input_deps=False,
relative_deps=False,
stdout=sys.stdout):
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
"path": {
"@type": "@id"
},
"format": {
"@type": "@id"
},
"id": "@id"
}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s",
json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
if print_input_deps:
printdeps(job_order_object,
loader,
stdout,
relative_deps,
basedir="file://%s/" % input_basedir)
return 0
return (job_order_object, input_basedir)
def load_job_order(args, t, parser):
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s", job_order_object)
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
return (job_order_object, input_basedir)
def load_job(job_file):
loader = Loader(load.jobloaderctx.copy())
job_order_object, _ = loader.resolve_ref(job_file, checklinks=False)
return job_order_object
def init_job_order(
job_order_object: Optional[CWLObjectType],
args: argparse.Namespace,
process: Process,
loader: Loader,
stdout: Union[TextIO, StreamWriter],
print_input_deps: bool = False,
relative_deps: str = "primary",
make_fs_access: Callable[[str], StdFsAccess] = StdFsAccess,
input_basedir: str = "",
secret_store: Optional[SecretStore] = None,
input_required: bool = True,
) -> CWLObjectType:
secrets_req, _ = process.get_requirement(
"http://commonwl.org/cwltool#Secrets")
if job_order_object is None:
namemap = {} # type: Dict[str, str]
records = [] # type: List[str]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow),
process,
namemap,
records,
input_required,
)
if args.tool_help:
toolparser.print_help()
exit(0)
cmd_line = vars(toolparser.parse_args(args.job_order))
for record_name in records:
record = {}
record_items = {
k: v
for k, v in cmd_line.items() if k.startswith(record_name)
}
for key, value in record_items.items():
record[key[len(record_name) + 1:]] = value
del cmd_line[key]
cmd_line[str(record_name)] = record
if "job_order" in cmd_line and cmd_line["job_order"]:
try:
job_order_object = cast(
CWLObjectType,
loader.resolve_ref(cmd_line["job_order"])[0],
)
except Exception:
_logger.exception("Failed to resolv job_order: %s",
cmd_line["job_order"])
exit(1)
else:
job_order_object = {"id": args.workflow}
del cmd_line["job_order"]
job_order_object.update({namemap[k]: v for k, v in cmd_line.items()})
if secret_store and secrets_req:
secret_store.store(
[
shortname(sc)
for sc in cast(List[str], secrets_req["secrets"])
],
job_order_object,
)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(
"Parsed job order from command line: %s",
json_dumps(job_order_object, indent=4),
)
for inp in process.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if job_order_object is None:
if process.tool["inputs"]:
if toolparser is not None:
print("\nOptions for {} ".format(args.workflow))
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
exit(1)
else:
job_order_object = {}
if print_input_deps:
basedir = None # type: Optional[str]
uri = cast(str, job_order_object["id"])
if uri == args.workflow:
basedir = os.path.dirname(uri)
uri = ""
printdeps(
job_order_object,
loader,
stdout,
relative_deps,
uri,
basedir=basedir,
nestdirs=False,
)
exit(0)
def path_to_loc(p: CWLObjectType) -> None:
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
ns = {} # type: ContextType
ns.update(cast(ContextType, job_order_object.get("$namespaces", {})))
ns.update(cast(ContextType, process.metadata.get("$namespaces", {})))
ld = Loader(ns)
def expand_formats(p: CWLObjectType) -> None:
if "format" in p:
p["format"] = ld.expand_url(cast(str, p["format"]), "")
visit_class(job_order_object, ("File", "Directory"), path_to_loc)
visit_class(
job_order_object,
("File", ),
functools.partial(add_sizes, make_fs_access(input_basedir)),
)
visit_class(job_order_object, ("File", ), expand_formats)
adjustDirObjs(job_order_object, trim_listing)
normalizeFilesDirs(job_order_object)
if secret_store and secrets_req:
secret_store.store(
[shortname(sc) for sc in cast(List[str], secrets_req["secrets"])],
job_order_object,
)
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return job_order_object
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
stdout=sys.stdout, make_fs_access=None):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Type[StdFsAccess]) -> Union[int, Tuple[Dict[Text, Any], Text]]
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
u"path": {u"@type": u"@id"},
u"location": {u"@type": u"@id"},
u"format": {u"@type": u"@id"},
u"id": u"@id"}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[Text, Text]
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print u"\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object, loader, stdout, relative_deps, "",
basedir=u"file://%s/" % input_basedir)
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
adjustDirObjs(job_order_object, pathToLoc)
adjustFileObjs(job_order_object, pathToLoc)
normalizeFilesDirs(job_order_object)
adjustDirObjs(job_order_object, cast(Callable[..., Any],
functools.partial(getListing, make_fs_access(input_basedir))))
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def pack(
document_loader: Loader,
processobj, # type: Union[Dict[str, Any], List[Dict[str, Any]]]
uri, # type: str
metadata, # type: Dict[str, str]
rewrite_out=None, # type: Optional[Dict[str, str]]
): # type: (...) -> Dict[str, Any]
document_loader = SubLoader(document_loader)
document_loader.idx = {}
if isinstance(processobj, MutableMapping):
document_loader.idx[processobj["id"]] = CommentedMap(
processobj.items())
elif isinstance(processobj, MutableSequence):
_, frag = urllib.parse.urldefrag(uri)
for po in processobj:
if not frag:
if po["id"].endswith("#main"):
uri = po["id"]
document_loader.idx[po["id"]] = CommentedMap(po.items())
document_loader.idx[metadata["id"]] = CommentedMap(metadata.items())
def loadref(base, uri):
# type: (Optional[str], str) -> Union[Dict[str, Any], List[Dict[str, Any]], str, None]
return document_loader.resolve_ref(uri, base_url=base)[0]
ids = set() # type: Set[str]
find_ids(processobj, ids)
runs = {uri}
find_run(processobj, loadref, runs)
for f in runs:
find_ids(document_loader.resolve_ref(f)[0], ids)
names = set() # type: Set[str]
if rewrite_out is None:
rewrite = {} # type: Dict[str, str]
else:
rewrite = rewrite_out
mainpath, _ = urllib.parse.urldefrag(uri)
def rewrite_id(r, mainuri):
# type: (str, str) -> None
if r == mainuri:
rewrite[r] = "#main"
elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"):
if r[len(mainuri):].startswith("#main/"):
rewrite[r] = "#" + uniquename(r[len(mainuri) + 1:], names)
else:
rewrite[r] = "#" + uniquename("main/" + r[len(mainuri) + 1:],
names)
else:
path, frag = urllib.parse.urldefrag(r)
if path == mainpath:
rewrite[r] = "#" + uniquename(frag, names)
else:
if path not in rewrite:
rewrite[path] = "#" + uniquename(shortname(path), names)
sortedids = sorted(ids)
for r in sortedids:
rewrite_id(r, uri)
packed = CommentedMap(
(("$graph", CommentedSeq()), ("cwlVersion", metadata["cwlVersion"])))
namespaces = metadata.get("$namespaces", None)
schemas = set() # type: Set[str]
if "$schemas" in metadata:
for each_schema in metadata["$schemas"]:
schemas.add(each_schema)
for r in sorted(runs):
dcr, metadata = document_loader.resolve_ref(r)
if isinstance(dcr, CommentedSeq):
dcr = dcr[0]
dcr = cast(CommentedMap, dcr)
if not isinstance(dcr, MutableMapping):
continue
metadata = cast(Dict[str, Any], metadata)
if "$schemas" in metadata:
for s in metadata["$schemas"]:
schemas.add(s)
if dcr.get("class") not in ("Workflow", "CommandLineTool",
"ExpressionTool"):
continue
dc = cast(Dict[str, Any], copy.deepcopy(dcr))
v = rewrite[r]
dc["id"] = v
for n in ("name", "cwlVersion", "$namespaces", "$schemas"):
if n in dc:
del dc[n]
packed["$graph"].append(dc)
if schemas:
packed["$schemas"] = list(schemas)
for r in list(rewrite.keys()):
v = rewrite[r]
replace_refs(packed, rewrite, r + "/" if "#" in r else r + "#",
v + "/")
import_embed(packed, set())
if len(packed["$graph"]) == 1:
# duplicate 'cwlVersion' and $schemas inside $graph when there is only
# a single item because we will print the contents inside '$graph'
# rather than whole dict
packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"]
if schemas:
packed["$graph"][0]["$schemas"] = list(schemas)
# always include $namespaces in the #main
if namespaces:
packed["$graph"][0]["$namespaces"] = namespaces
return packed