def test_scoped_id() -> None:
ldr = Loader({})
ctx = {
"id": "@id",
"location": {"@id": "@id", "@type": "@id"},
"bar": "http://example.com/bar",
"ex": "http://example.com/",
} # type: ContextType
ldr.add_context(ctx)
ra, _ = ldr.resolve_all(
cmap({"id": "foo", "bar": {"id": "baz"}}), "http://example.com"
)
assert {
"id": "http://example.com/#foo",
"bar": {"id": "http://example.com/#foo/baz"},
} == ra
g = makerdf(None, ra, ctx)
print(g.serialize(format="n3"))
ra, _ = ldr.resolve_all(
cmap({"location": "foo", "bar": {"location": "baz"}}),
"http://example.com",
checklinks=False,
)
assert {
"location": "http://example.com/foo",
"bar": {"location": "http://example.com/baz"},
} == ra
g = makerdf(None, ra, ctx)
print(g.serialize(format="n3"))
ra, _ = ldr.resolve_all(
cmap({"id": "foo", "bar": {"location": "baz"}}),
"http://example.com",
checklinks=False,
)
assert {
"id": "http://example.com/#foo",
"bar": {"location": "http://example.com/baz"},
} == ra
g = makerdf(None, ra, ctx)
print(g.serialize(format="n3"))
ra, _ = ldr.resolve_all(
cmap({"location": "foo", "bar": {"id": "baz"}}),
"http://example.com",
checklinks=False,
)
assert {
"location": "http://example.com/foo",
"bar": {"id": "http://example.com/#baz"},
} == ra
g = makerdf(None, ra, ctx)
print(g.serialize(format="n3"))
def test_secondaryFile_dsl_ref() -> None:
ldr = Loader({})
ldr.add_context({"secondaryFiles": {"secondaryFilesDSL": True}})
ra, _ = ldr.resolve_all(cmap({"secondaryFiles": ".foo"}), "")
assert {"secondaryFiles": {"pattern": ".foo", "required": None}} == ra
ra, _ = ldr.resolve_all(cmap({"secondaryFiles": ".foo?"}), "")
assert {"secondaryFiles": {"pattern": ".foo", "required": False}} == ra
ra, _ = ldr.resolve_all(cmap({"secondaryFiles": [".foo"]}), "")
assert {"secondaryFiles": [{"pattern": ".foo", "required": None}]} == ra
ra, _ = ldr.resolve_all(cmap({"secondaryFiles": [".foo?"]}), "")
assert {"secondaryFiles": [{"pattern": ".foo", "required": False}]} == ra
def load_job_order(
args: argparse.Namespace,
stdin: IO[Any],
fetcher_constructor: Optional[FetcherCallableType],
overrides_list: List[CWLObjectType],
tool_file_uri: str,
) -> Tuple[Optional[CWLObjectType], str, Loader]:
job_order_object = None
job_order_file = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.main.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(
job_order_object, file_uri(os.getcwd()) + "/"
)
else:
job_order_file = None
if job_order_object is not None:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file is not None:
input_basedir = (
args.basedir
if args.basedir
else os.path.abspath(os.path.dirname(job_order_file))
)
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
if (
job_order_object is not None
and "http://commonwl.org/cwltool#overrides" in job_order_object
):
ov_uri = file_uri(job_order_file or input_basedir)
overrides_list.extend(
resolve_overrides(job_order_object, ov_uri, tool_file_uri)
)
del job_order_object["http://commonwl.org/cwltool#overrides"]
if job_order_object is None:
input_basedir = args.basedir if args.basedir else os.getcwd()
if job_order_object is not None and not isinstance(
job_order_object, MutableMapping
):
_logger.error(
"CWL input object at %s is not formatted correctly, it should be a "
"JSON/YAML dictionay, not %s.\n"
"Raw input object:\n%s",
job_order_file or "stdin",
type(job_order_object),
job_order_object,
)
sys.exit(1)
return (job_order_object, input_basedir, loader)
def test_idmap() -> None:
ldr = Loader({})
ldr.add_context(
{
"inputs": {
"@id": "http://example.com/inputs",
"mapSubject": "id",
"mapPredicate": "a",
},
"outputs": {"@type": "@id", "identity": True},
"id": "@id",
}
)
ra, _ = ldr.resolve_all(
cmap(
{
"id": "stuff",
"inputs": {"zip": 1, "zing": 2},
"outputs": ["out"],
"other": {"n": 9},
}
),
"http://example2.com/",
)
assert isinstance(ra, CommentedMap)
assert "http://example2.com/#stuff" == ra["id"]
for item in ra["inputs"]:
if item["a"] == 2:
assert "http://example2.com/#stuff/zing" == item["id"]
else:
assert "http://example2.com/#stuff/zip" == item["id"]
assert ["http://example2.com/#stuff/out"] == ra["outputs"]
assert {"n": 9} == ra["other"]
def cwl_dispatch(self, json):
try:
cwlwf, it_is_workflow = load_cwl(self.dag.default_args["cwl_workflow"], self.dag.default_args)
cwl_context = {
"outdir": mkdtemp(
prefix=os.path.abspath(os.path.join(self.tmp_folder, 'dag_tmp_')))
}
_jobloaderctx = jobloaderctx.copy()
_jobloaderctx.update(cwlwf.metadata.get("$namespaces", {}))
loader = Loader(_jobloaderctx)
try:
job_order_object = yaml.round_trip_load(io.StringIO(initial_value=dumps(json)))
job_order_object, _ = loader.resolve_all(job_order_object,
file_uri(os.getcwd()) + "/",
checklinks=False)
except Exception as e:
_logger.error("Job Loader: {}".format(str(e)))
job_order_object = init_job_order(job_order_object, None, cwlwf, loader, sys.stdout)
cwl_context['promises'] = job_order_object
logging.info(
'{0}: Final job: \n {1}'.format(self.task_id, dumps(cwl_context, indent=4)))
return cwl_context
except Exception as e:
_logger.info(
'Dispatch Exception {0}: \n {1} {2}'.format(self.task_id, type(e), e))
pass
return None
def load_job_order(args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
):
# type: (...) -> Tuple[Dict[Text, Any], Text, Loader]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object:
overrides.extend(resolve_overrides(job_order_object, file_uri(job_order_file), tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if not job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
return (job_order_object, input_basedir, loader)
def test_rdf_datetime() -> None:
"""Affirm that datetime objects can be serialized in makerdf()."""
ldr = Loader({})
ctx: ContextType = {
"id": "@id",
"location": {"@id": "@id", "@type": "@id"},
"bar": "http://example.com/bar",
"ex": "http://example.com/",
}
ldr.add_context(ctx)
ra: CommentedMap = cast(
CommentedMap,
ldr.resolve_all(
cmap(
{
"id": "foo",
"bar": {"id": "baz"},
}
),
"http://example.com",
)[0],
)
ra["s:dateCreated"] = datetime.datetime(2020, 10, 8)
g = makerdf(None, ra, ctx)
g.serialize(destination=stdout(), format="n3")
g2 = makerdf(None, CommentedSeq([ra]), ctx)
g2.serialize(destination=stdout(), format="n3")
def resolve_overrides(ov, ov_uri, baseurl): # type: (CommentedMap, Text, Text) -> List[Dict[Text, Any]]
ovloader = Loader(overrides_ctx)
ret, _ = ovloader.resolve_all(ov, baseurl)
if not isinstance(ret, CommentedMap):
raise Exception("Expected CommentedMap, got %s" % type(ret))
cwl_docloader = get_schema("v1.0")[0]
cwl_docloader.resolve_all(ret, ov_uri)
return ret["overrides"]
def resolve_overrides(
ov, ov_uri,
baseurl): # type: (CommentedMap, Text, Text) -> List[Dict[Text, Any]]
ovloader = Loader(overrides_ctx)
ret, _ = ovloader.resolve_all(ov, baseurl)
if not isinstance(ret, CommentedMap):
raise Exception("Expected CommentedMap, got %s" % type(ret))
cwl_docloader = get_schema("v1.0")[0]
cwl_docloader.resolve_all(ret, ov_uri)
return ret["overrides"]
def resolve_overrides(ov, # Type: CommentedMap
ov_uri, # Type: Text
baseurl # type: Text
): # type: (...) -> List[Dict[Text, Any]]
ovloader = Loader(overrides_ctx)
ret, _ = ovloader.resolve_all(ov, baseurl)
if not isinstance(ret, CommentedMap):
raise Exception("Expected CommentedMap, got %s" % type(ret))
cwl_docloader = get_schema("v1.0")[0]
cwl_docloader.resolve_all(ret, ov_uri)
return ret["http://commonwl.org/cwltool#overrides"]
def resolve_overrides(ov, # Type: CommentedMap
ov_uri, # Type: Text
baseurl # type: Text
): # type: (...) -> List[Dict[Text, Any]]
ovloader = Loader(overrides_ctx)
ret, _ = ovloader.resolve_all(ov, baseurl)
if not isinstance(ret, CommentedMap):
raise Exception("Expected CommentedMap, got %s" % type(ret))
cwl_docloader = get_schema("v1.0")[0]
cwl_docloader.resolve_all(ret, ov_uri)
return ret["http://commonwl.org/cwltool#overrides"]
def resolve_overrides(
ov: IdxResultType,
ov_uri: str,
baseurl: str,
) -> List[CWLObjectType]:
ovloader = Loader(overrides_ctx)
ret, _ = ovloader.resolve_all(ov, baseurl)
if not isinstance(ret, CommentedMap):
raise Exception("Expected CommentedMap, got %s" % type(ret))
cwl_docloader = get_schema("v1.0")[0]
cwl_docloader.resolve_all(ret, ov_uri)
return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"])
def test_blank_node_id() -> None:
# Test that blank nodes are passed through and not considered
# relative paths. Blank nodes (also called anonymous ids) are
# URIs starting with "_:". They are randomly generated
# placeholders mainly used internally where an id is needed but
# was not given.
ldr = Loader({})
ctx = {"id": "@id"} # type: ContextType
ldr.add_context(ctx)
ra, _ = ldr.resolve_all(cmap({"id": "_:foo"}), "http://example.com")
assert {"id": "_:foo"} == ra
def test_typedsl_ref() -> None:
ldr = Loader({})
ldr.add_context(
{
"File": "http://example.com/File",
"null": "http://example.com/null",
"array": "http://example.com/array",
"type": {"@type": "@vocab", "typeDSL": True},
}
)
ra, _ = ldr.resolve_all(cmap({"type": "File"}), "")
assert {"type": "File"} == ra
ra, _ = ldr.resolve_all(cmap({"type": "File?"}), "")
assert {"type": ["null", "File"]} == ra
ra, _ = ldr.resolve_all(cmap({"type": "File[]"}), "")
assert {"type": {"items": "File", "type": "array"}} == ra
ra, _ = ldr.resolve_all(cmap({"type": "File[]?"}), "")
assert {"type": ["null", {"items": "File", "type": "array"}]} == ra
def load_job_order(
args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides_list, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader]
job_order_object = None
job_order_file = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx,
fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object,
file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object is not None:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file is not None:
input_basedir = args.basedir if args.basedir \
else os.path.abspath(os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object:
ov_uri = file_uri(job_order_file or input_basedir)
overrides_list.extend(
resolve_overrides(job_order_object, ov_uri, tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if job_order_object is None:
input_basedir = args.basedir if args.basedir else os.getcwd()
if job_order_object is not None and not isinstance(job_order_object,
MutableMapping):
_logger.error(
'CWL input object at %s is not formatted correctly, it should be a '
'JSON/YAML dictionay, not %s.\n'
'Raw input object:\n%s', job_order_file or "stdin",
type(job_order_object), job_order_object)
sys.exit(1)
return (job_order_object, input_basedir, loader)
def test_subscoped_id() -> None:
ldr = Loader({})
ctx = {
"id": "@id",
"bar": {"subscope": "bar"},
} # type: ContextType
ldr.add_context(ctx)
ra, _ = ldr.resolve_all(
cmap({"id": "foo", "bar": {"id": "baz"}}), "http://example.com"
)
assert {
"id": "http://example.com/#foo",
"bar": {"id": "http://example.com/#foo/bar/baz"},
} == ra
def test_mixin() -> None:
base_url = file_uri(os.path.join(os.getcwd(), "tests"))
ldr = Loader({})
path = get_data("tests/mixin.yml")
assert path
ra = ldr.resolve_ref(cmap({"$mixin": path, "one": "five"}), base_url=base_url)
assert {"id": "four", "one": "five"} == ra[0]
ldr = Loader({"id": "@id"})
ra = ldr.resolve_all(
cmap([{"id": "a", "m": {"$mixin": path}}, {"id": "b", "m": {"$mixin": path}}]),
base_url=base_url,
)
assert [
{"id": base_url + "#a", "m": {"id": base_url + "#a/four", "one": "two"}},
{"id": base_url + "#b", "m": {"id": base_url + "#b/four", "one": "two"}},
] == ra[0]
def load_job_order(args, # type: argparse.Namespace
stdin, # type: IO[Any]
fetcher_constructor, # Fetcher
overrides_list, # type: List[Dict[Text, Any]]
tool_file_uri # type: Text
): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader]
job_order_object = None
job_order_file = None
_jobloaderctx = jobloaderctx.copy()
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object is not None:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file is not None:
input_basedir = args.basedir if args.basedir \
else os.path.abspath(os.path.dirname(job_order_file))
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object:
ov_uri = file_uri(job_order_file or input_basedir)
overrides_list.extend(
resolve_overrides(job_order_object, ov_uri, tool_file_uri))
del job_order_object["http://commonwl.org/cwltool#overrides"]
if job_order_object is None:
input_basedir = args.basedir if args.basedir else os.getcwd()
if job_order_object is not None and not isinstance(job_order_object, MutableMapping):
_logger.error(
'CWL input object at %s is not formatted correctly, it should be a '
'JSON/YAML dictionay, not %s.\n'
'Raw input object:\n%s', job_order_file or "stdin",
type(job_order_object), job_order_object)
sys.exit(1)
return (job_order_object, input_basedir, loader)
def test_schemas() -> None:
loader = Loader({})
path = get_data("tests/EDAM.owl")
assert path
ra, _ = loader.resolve_all(
cmap(
{
"$schemas": [file_uri(path)],
"$namespaces": {"edam": "http://edamontology.org/"},
"edam:has_format": "edam:format_1915",
}
),
"",
)
assert {
"$schemas": [file_uri(path)],
"$namespaces": {"edam": "http://edamontology.org/"},
"http://edamontology.org/has_format": "http://edamontology.org/format_1915",
} == ra
def validate_document(
document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None, # type: FetcherConstructorType
skip_schemas=None, # type: bool
overrides=None, # type: List[Dict]
metadata=None, # type: Optional[Dict]
):
# type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, list):
workflowobj = cmap({"$graph": workflowobj}, fn=uri)
if not isinstance(workflowobj, dict):
raise ValueError("workflowjobj must be a dict, got '%s': %s" %
(type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
job_loader = default_loader(fetcher_constructor) # type: ignore
jobobj, _ = job_loader.resolve_all(workflowobj, uri)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
if "http://commonwl.org/cwltool#overrides" in jobobj:
overrides.extend(resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
workflowobj = fetch_document(
uri, fetcher_constructor=fetcher_constructor)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
if "cwlVersion" not in workflowobj:
if metadata and 'cwlVersion' in metadata:
workflowobj['cwlVersion'] = metadata['cwlVersion']
else:
raise ValidationException(
"No cwlVersion found."
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>")
if not isinstance(workflowobj["cwlVersion"], (str, Text)):
raise Exception("'cwlVersion' must be a string, got %s" %
type(workflowobj["cwlVersion"]))
# strip out version
workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)",
"", workflowobj["cwlVersion"])
if workflowobj["cwlVersion"] not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = list(ALLUPDATES) # ALLUPDATES is a dict
versions.sort()
raise ValidationException(
"'cwlVersion' not valid. Supported CWL versions are: \n{}".format(
"\n".join(versions)))
if workflowobj["cwlVersion"] == "draft-2":
workflowobj = cast(
CommentedMap,
cmap(
update._draft2toDraft3dev1(workflowobj,
document_loader,
uri,
update_steps=False)))
if "@graph" in workflowobj:
workflowobj["$graph"] = workflowobj["@graph"]
del workflowobj["@graph"]
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text]
document_loader = Loader(sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=document_loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor,
skip_schemas=skip_schemas)
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, new_metadata = document_loader.resolve_all(
workflowobj, fileuri)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not new_metadata:
if not isinstance(processobj, dict):
raise ValidationException("Draft-2 workflows must be a dict.")
new_metadata = cast(
CommentedMap,
cmap(
{
"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]
},
fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, new_metadata, uri
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if new_metadata.get("cwlVersion") != update.LATEST:
processobj = cast(
CommentedMap,
cmap(
update.update(processobj, document_loader, fileuri, enable_dev,
new_metadata)))
if jobobj:
new_metadata[u"cwl:defaults"] = jobobj
if overrides:
new_metadata[u"cwltool:overrides"] = overrides
return document_loader, avsc_names, processobj, new_metadata, uri
def validate_document(
document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None, # type: FetcherConstructorType
skip_schemas=None, # type: bool
overrides=None, # type: List[Dict]
metadata=None, # type: Optional[Dict]
do_validate=True):
# type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, list):
workflowobj = cmap({"$graph": workflowobj}, fn=uri)
if not isinstance(workflowobj, dict):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
job_loader = default_loader(fetcher_constructor) # type: ignore
jobobj, _ = job_loader.resolve_all(workflowobj,
uri,
checklinks=do_validate)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
if "http://commonwl.org/cwltool#overrides" in jobobj:
overrides.extend(resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
workflowobj = fetch_document(
uri, fetcher_constructor=fetcher_constructor)[1]
def var_spool_cwl_detector(
obj, # type: Union[Mapping, Iterable, Text]
item=None, # type: Optional[Any]
obj_key=None, # type: Optional[Any]
): # type: (...)->None
""" Detects any textual reference to /var/spool/cwl. """
if isinstance(obj, string_types):
if "var/spool/cwl" in obj:
message = SourceLine(
item=item,
key=obj_key,
raise_type=Text,
include_traceback=_logger.isEnabledFor(
logging.DEBUG)).makeError(
"Non-portable reference to /var/spool/cwl found: "
"'{}'.\n Replace with /var/spool/cwl/ with "
"$(runtime.outdir).".format(obj))
if not strict:
_logger.warning(message)
else:
raise ValidationException(message)
else:
return
elif isinstance(obj, Mapping):
for key, value in iteritems(obj):
var_spool_cwl_detector(value, obj, key)
elif isinstance(obj, Iterable):
for element in obj:
var_spool_cwl_detector(element, obj, None)
var_spool_cwl_detector(workflowobj)
fileuri = urllib.parse.urldefrag(uri)[0]
if "cwlVersion" not in workflowobj:
if metadata and 'cwlVersion' in metadata:
workflowobj['cwlVersion'] = metadata['cwlVersion']
else:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(workflowobj["cwlVersion"], (str, Text)):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(
type(workflowobj["cwlVersion"])))
# strip out version
workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)",
"", workflowobj["cwlVersion"])
if workflowobj["cwlVersion"] not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text]
document_loader = Loader(sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=document_loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor,
skip_schemas=skip_schemas)
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, new_metadata = document_loader.resolve_all(
workflowobj, fileuri, checklinks=do_validate)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not new_metadata:
new_metadata = cast(
CommentedMap,
cmap(
{
"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]
},
fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, new_metadata, uri
if do_validate:
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if new_metadata.get("cwlVersion") != update.LATEST:
processobj = cast(
CommentedMap,
cmap(
update.update(processobj, document_loader, fileuri, enable_dev,
new_metadata)))
if jobobj:
new_metadata[u"cwl:defaults"] = jobobj
if overrides:
new_metadata[u"cwltool:overrides"] = overrides
return document_loader, avsc_names, processobj, new_metadata, uri
def resolve_and_validate_document(
loadingContext,
workflowobj,
uri,
preprocess_only=False, # type: bool
skip_schemas=None, # type: bool
):
# type: (...) -> Tuple[LoadingContext, Text]
"""Validate a CWL document."""
loadingContext = loadingContext.copy()
if not isinstance(workflowobj, MutableMapping):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
jobobj, _ = loadingContext.loader.resolve_all(
workflowobj, uri, checklinks=loadingContext.do_validate)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
workflowobj = fetch_document(uri, loadingContext)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
cwlVersion = workflowobj.get("cwlVersion")
if not cwlVersion:
fileobj = fetch_document(fileuri, loadingContext)[1]
cwlVersion = fileobj.get("cwlVersion")
if not cwlVersion:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(cwlVersion, string_types):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(type(cwlVersion)))
# strip out version
cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion)
if cwlVersion not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
if isinstance(jobobj, CommentedMap
) and "http://commonwl.org/cwltool#overrides" in jobobj:
loadingContext.overrides_list.extend(
resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
if isinstance(jobobj, CommentedMap
) and "https://w3id.org/cwl/cwl#requirements" in jobobj:
if cwlVersion not in ("v1.1.0-dev1", ):
raise ValidationException(
"`cwl:requirements` in the input object is not part of CWL "
"v1.0. You can adjust to use `cwltool:overrides` instead; or you "
"can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
"--enable-dev.")
loadingContext.overrides_list.append({
"overrideTarget":
uri,
"requirements":
jobobj["https://w3id.org/cwl/cwl#requirements"]
})
del jobobj["https://w3id.org/cwl/cwl#requirements"]
(sch_document_loader, avsc_names) = \
process.get_schema(cwlVersion)[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None]
document_loader = Loader(
sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=loadingContext.loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=loadingContext.fetcher_constructor,
skip_schemas=skip_schemas)
if cwlVersion == "v1.0":
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, metadata = document_loader.resolve_all(
workflowobj, fileuri, checklinks=loadingContext.do_validate)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException(
"Workflow must be a CommentedMap or CommentedSeq.")
if not isinstance(metadata, CommentedMap):
raise ValidationException("metadata must be a CommentedMap, was %s" %
type(metadata))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return loadingContext, uri
if loadingContext.do_validate:
schema.validate_doc(avsc_names, processobj, document_loader,
loadingContext.strict)
if loadingContext.do_update:
processobj = cast(
CommentedMap,
cmap(
update.update(processobj, document_loader, fileuri,
loadingContext.enable_dev, metadata)))
if isinstance(processobj, MutableMapping):
document_loader.idx[processobj["id"]] = processobj
elif isinstance(processobj, MutableSequence):
document_loader.idx[metadata["id"]] = metadata
for po in processobj:
document_loader.idx[po["id"]] = po
if jobobj is not None:
loadingContext.jobdefaults = jobobj
loadingContext.loader = document_loader
loadingContext.avsc_names = avsc_names
loadingContext.metadata = metadata
return loadingContext, uri
def resolve_and_validate_document(loadingContext,
workflowobj,
uri,
preprocess_only=False, # type: bool
skip_schemas=None, # type: bool
):
# type: (...) -> Tuple[LoadingContext, Text]
"""Validate a CWL document."""
loadingContext = loadingContext.copy()
if not isinstance(workflowobj, MutableMapping):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
jobobj, _ = loadingContext.loader.resolve_all(workflowobj, uri, checklinks=loadingContext.do_validate)
uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
workflowobj = fetch_document(uri, loadingContext)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
cwlVersion = workflowobj.get("cwlVersion")
if not cwlVersion:
fileobj = fetch_document(fileuri, loadingContext)[1]
cwlVersion = fileobj.get("cwlVersion")
if not cwlVersion:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(cwlVersion, string_types):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(
type(cwlVersion)))
# strip out version
cwlVersion = re.sub(
r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
cwlVersion)
if cwlVersion not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj:
loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj:
if cwlVersion not in ("v1.1.0-dev1",):
raise ValidationException(
"`cwl:requirements` in the input object is not part of CWL "
"v1.0. You can adjust to use `cwltool:overrides` instead; or you "
"can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
"--enable-dev.")
loadingContext.overrides_list.append({"overrideTarget": uri,
"requirements": jobobj["https://w3id.org/cwl/cwl#requirements"]})
del jobobj["https://w3id.org/cwl/cwl#requirements"]
(sch_document_loader, avsc_names) = \
process.get_schema(cwlVersion)[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None]
document_loader = Loader(sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=loadingContext.loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=loadingContext.fetcher_constructor,
skip_schemas=skip_schemas)
if cwlVersion == "v1.0":
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, metadata = document_loader.resolve_all(
workflowobj, fileuri, checklinks=loadingContext.do_validate)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.")
if not isinstance(metadata, CommentedMap):
raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return loadingContext, uri
if loadingContext.do_validate:
schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict)
if loadingContext.do_update:
processobj = cast(CommentedMap, cmap(update.update(
processobj, document_loader, fileuri, loadingContext.enable_dev, metadata)))
if isinstance(processobj, MutableMapping):
document_loader.idx[processobj["id"]] = processobj
elif isinstance(processobj, MutableSequence):
document_loader.idx[metadata["id"]] = metadata
for po in processobj:
document_loader.idx[po["id"]] = po
if jobobj is not None:
loadingContext.jobdefaults = jobobj
loadingContext.loader = document_loader
loadingContext.avsc_names = avsc_names
loadingContext.metadata = metadata
return loadingContext, uri
def validate_document(
document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None
# type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]
):
# type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, list):
workflowobj = {"$graph": workflowobj}
if not isinstance(workflowobj, dict):
raise ValueError("workflowjobj must be a dict, got '%s': %s" %
(type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
jobobj, _ = document_loader.resolve_all(workflowobj, uri)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
workflowobj = fetch_document(
uri, fetcher_constructor=fetcher_constructor)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
if "cwlVersion" in workflowobj:
if not isinstance(workflowobj["cwlVersion"], (str, Text)):
raise Exception("'cwlVersion' must be a string, got %s" %
type(workflowobj["cwlVersion"]))
workflowobj["cwlVersion"] = re.sub(
r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
workflowobj["cwlVersion"])
else:
_logger.warn("No cwlVersion found, treating this file as draft-2.")
workflowobj["cwlVersion"] = "draft-2"
if workflowobj["cwlVersion"] == "draft-2":
workflowobj = cast(
CommentedMap,
cmap(
update._draft2toDraft3dev1(workflowobj,
document_loader,
uri,
update_steps=False)))
if "@graph" in workflowobj:
workflowobj["$graph"] = workflowobj["@graph"]
del workflowobj["@graph"]
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, unicode]
document_loader = Loader(sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=document_loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor)
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not metadata:
if not isinstance(processobj, dict):
raise ValidationException("Draft-2 workflows must be a dict.")
metadata = cast(
CommentedMap,
cmap(
{
"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]
},
fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, metadata, uri
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if metadata.get("cwlVersion") != update.LATEST:
processobj = cast(
CommentedMap,
cmap(
update.update(processobj, document_loader, fileuri, enable_dev,
metadata)))
if jobobj:
metadata[u"cwl:defaults"] = jobobj
return document_loader, avsc_names, processobj, metadata, uri
def test_scoped_ref() -> None:
ldr = Loader({})
ldr.add_context({
"scatter": {
"@type": "@id",
"refScope": 0
},
"source": {
"@type": "@id",
"refScope": 2
},
"in": {
"mapSubject": "id",
"mapPredicate": "source"
},
"out": {
"@type": "@id",
"identity": True
},
"inputs": {
"mapSubject": "id",
"mapPredicate": "type"
},
"outputs": {
"mapSubject": "id"
},
"steps": {
"mapSubject": "id"
},
"id": "@id",
})
ra, _ = ldr.resolve_all(
cmap({
"inputs": {
"inp": "string",
"inp2": "string"
},
"outputs": {
"out": {
"type": "string",
"source": "step2/out"
}
},
"steps": {
"step1": {
"in": {
"inp": "inp",
"inp2": "#inp2",
"inp3": ["inp", "inp2"]
},
"out": ["out"],
"scatter": "inp",
},
"step2": {
"in": {
"inp": "step1/out"
},
"scatter": "inp",
"out": ["out"],
},
},
}),
"http://example2.com/",
)
assert {
"inputs": [
{
"id": "http://example2.com/#inp",
"type": "string"
},
{
"id": "http://example2.com/#inp2",
"type": "string"
},
],
"outputs": [{
"id": "http://example2.com/#out",
"type": "string",
"source": "http://example2.com/#step2/out",
}],
"steps": [
{
"id":
"http://example2.com/#step1",
"scatter":
"http://example2.com/#step1/inp",
"in": [
{
"id": "http://example2.com/#step1/inp",
"source": "http://example2.com/#inp",
},
{
"id": "http://example2.com/#step1/inp2",
"source": "http://example2.com/#inp2",
},
{
"id":
"http://example2.com/#step1/inp3",
"source": [
"http://example2.com/#inp",
"http://example2.com/#inp2",
],
},
],
"out": ["http://example2.com/#step1/out"],
},
{
"id":
"http://example2.com/#step2",
"scatter":
"http://example2.com/#step2/inp",
"in": [{
"id": "http://example2.com/#step2/inp",
"source": "http://example2.com/#step1/out",
}],
"out": ["http://example2.com/#step2/out"],
},
],
} == ra
def validate_document(
document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
overrides, # type: List[Dict]
metadata, # type: Dict[Text, Any]
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None, # type: FetcherConstructorType
skip_schemas=None, # type: bool
do_validate=True # type: bool
):
# type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, MutableSequence):
workflowobj = cmap({"$graph": workflowobj}, fn=uri)
if not isinstance(workflowobj, MutableMapping):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
job_loader = default_loader(fetcher_constructor) # type: ignore
jobobj, _ = job_loader.resolve_all(workflowobj,
uri,
checklinks=do_validate)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
if isinstance(jobobj, CommentedMap
) and "http://commonwl.org/cwltool#overrides" in jobobj:
overrides.extend(resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
workflowobj = fetch_document(
uri, fetcher_constructor=fetcher_constructor)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
if "cwlVersion" not in workflowobj:
if 'cwlVersion' in metadata:
workflowobj['cwlVersion'] = metadata['cwlVersion']
else:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(workflowobj["cwlVersion"], string_types):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(
type(workflowobj["cwlVersion"])))
# strip out version
workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)",
"", workflowobj["cwlVersion"])
if workflowobj["cwlVersion"] not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None]
document_loader = Loader(sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=document_loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor,
skip_schemas=skip_schemas)
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, new_metadata = document_loader.resolve_all(
workflowobj, fileuri, checklinks=do_validate)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not new_metadata and isinstance(processobj, CommentedMap):
new_metadata = cast(
CommentedMap,
cmap(
{
"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]
},
fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, new_metadata, uri
if do_validate:
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if new_metadata.get("cwlVersion") != update.LATEST:
processobj = cast(
CommentedMap,
cmap(
update.update(processobj, document_loader, fileuri, enable_dev,
new_metadata)))
if jobobj is not None:
new_metadata[u"cwl:defaults"] = jobobj
if overrides:
new_metadata[u"cwltool:overrides"] = overrides
return document_loader, avsc_names, processobj, new_metadata, uri
def main(args=None,
executor=single_job_executor,
makeTool=workflow.defaultMakeTool,
parser=None,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr):
_logger.removeHandler(defaultStreamHandler)
_logger.addHandler(logging.StreamHandler(stderr))
if args is None:
args = sys.argv[1:]
if parser is None:
parser = arg_parser()
args = parser.parse_args(args)
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
_logger.setLevel(logging.DEBUG)
pkg = pkg_resources.require("cwltool")
if pkg:
if args.version:
print "%s %s" % (sys.argv[0], pkg[0].version)
return 0
else:
_logger.info("%s %s", sys.argv[0], pkg[0].version)
if not args.workflow:
parser.print_help()
_logger.error("")
_logger.error("CWL document required")
return 1
t = load_tool(args.workflow, args.update, args.strict, makeTool, args.debug, args.print_pre)
if type(t) == int:
return t
if args.print_rdf:
printrdf(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.print_dot:
printdot(args.workflow, processobj, ctx, args.rdf_serializer)
return 0
if args.tmp_outdir_prefix != 'tmp':
# Use user defined temp directory (if it exists)
args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix)
if not os.path.exists(args.tmp_outdir_prefix):
_logger.error("Intermediate output directory prefix doesn't exist, reverting to default")
return 1
if args.tmpdir_prefix != 'tmp':
# Use user defined prefix (if the folder exists)
args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix)
if not os.path.exists(args.tmpdir_prefix):
_logger.error("Temporary directory prefix doesn't exist.")
return 1
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
loader = Loader({"id": "@id", "path": {"@type": "@id"}})
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s", job_order_object)
else:
job_order_object = None
if not job_order_object:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
try:
out = executor(t, job_order_object,
input_basedir, args,
conformance_test=args.conformance_test,
dry_run=args.dry_run,
outdir=args.outdir,
tmp_outdir_prefix=args.tmp_outdir_prefix,
use_container=args.use_container,
preserve_environment=args.preserve_environment,
pull_image=args.enable_pull,
rm_container=args.rm_container,
tmpdir_prefix=args.tmpdir_prefix,
rm_tmpdir=args.rm_tmpdir,
makeTool=makeTool,
move_outputs=args.move_outputs
)
# This is the workflow output, it needs to be written
stdout.write(json.dumps(out, indent=4))
stdout.flush()
except (validate.ValidationException) as e:
_logger.error("Input object failed validation:\n%s", e, exc_info=(e if args.debug else False))
return 1
except workflow.WorkflowException as e:
_logger.error("Workflow error:\n %s", e, exc_info=(e if args.debug else False))
return 1
return 0
def validate_document(document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
overrides, # type: List[Dict]
metadata, # type: Dict[Text, Any]
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None, # type: FetcherConstructorType
skip_schemas=None, # type: bool
do_validate=True # type: bool
):
# type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, MutableSequence):
workflowobj = cmap({
"$graph": workflowobj
}, fn=uri)
if not isinstance(workflowobj, MutableMapping):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
job_loader = default_loader(fetcher_constructor) # type: ignore
jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate)
uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj:
overrides.extend(resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
if "cwlVersion" not in workflowobj:
if 'cwlVersion' in metadata:
workflowobj['cwlVersion'] = metadata['cwlVersion']
else:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(workflowobj["cwlVersion"], string_types):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(
type(workflowobj["cwlVersion"])))
# strip out version
workflowobj["cwlVersion"] = re.sub(
r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
workflowobj["cwlVersion"])
if workflowobj["cwlVersion"] not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None]
document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
idx=document_loader.idx, cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas)
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, new_metadata = document_loader.resolve_all(
workflowobj, fileuri, checklinks=do_validate)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not new_metadata and isinstance(processobj, CommentedMap):
new_metadata = cast(CommentedMap, cmap(
{"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]}, fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, new_metadata, uri
if do_validate:
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if new_metadata.get("cwlVersion") != update.LATEST:
processobj = cast(CommentedMap, cmap(update.update(
processobj, document_loader, fileuri, enable_dev, new_metadata)))
if jobobj is not None:
new_metadata[u"cwl:defaults"] = jobobj
if overrides:
new_metadata[u"cwltool:overrides"] = overrides
return document_loader, avsc_names, processobj, new_metadata, uri
def load_job_order(args, t, parser):
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s", job_order_object)
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
return (job_order_object, input_basedir)
def validate_document(document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]
):
# type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, list):
workflowobj = {
"$graph": workflowobj
}
if not isinstance(workflowobj, dict):
raise ValueError("workflowjobj must be a dict")
jobobj = None
if "cwl:tool" in workflowobj:
jobobj, _ = document_loader.resolve_all(workflowobj, uri)
uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]
fileuri = urlparse.urldefrag(uri)[0]
if "cwlVersion" in workflowobj:
if not isinstance(workflowobj["cwlVersion"], (str, Text)):
raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
workflowobj["cwlVersion"] = re.sub(
r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
workflowobj["cwlVersion"])
else:
_logger.warn("No cwlVersion found, treating this file as draft-2.")
workflowobj["cwlVersion"] = "draft-2"
if workflowobj["cwlVersion"] == "draft-2":
workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1(
workflowobj, document_loader, uri, update_steps=False)))
if "@graph" in workflowobj:
workflowobj["$graph"] = workflowobj["@graph"]
del workflowobj["@graph"]
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, unicode]
document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
idx=document_loader.idx, cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor)
workflowobj["id"] = fileuri
processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not metadata:
if not isinstance(processobj, dict):
raise ValidationException("Draft-2 workflows must be a dict.")
metadata = cast(CommentedMap, cmap({"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]},
fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, metadata, uri
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if metadata.get("cwlVersion") != update.LATEST:
processobj = cast(CommentedMap, cmap(update.update(
processobj, document_loader, fileuri, enable_dev, metadata)))
if jobobj:
metadata[u"cwl:defaults"] = jobobj
return document_loader, avsc_names, processobj, metadata, uri
def load_job_order(args,
t,
stdin,
print_input_deps=False,
relative_deps=False,
stdout=sys.stdout,
make_fs_access=None,
fetcher_constructor=None):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[Text, Text], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
_jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(_jobloaderctx,
fetcher_constructor=fetcher_constructor) # type: ignore
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object,
file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[Text, Text]
records = [] # type: List[Text]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap, records)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
for record_name in records:
record = {}
record_items = {
k: v
for k, v in six.iteritems(cmd_line)
if k.startswith(record_name)
}
for key, value in six.iteritems(record_items):
record[key[len(record_name) + 1:]] = value
del cmd_line[key]
cmd_line[str(record_name)] = record
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
else:
job_order_object = {"id": args.workflow}
del cmd_line["job_order"]
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Parsed job order from command line: %s",
json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print(u"\nOptions for {} ".format(args.workflow))
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object,
loader,
stdout,
relative_deps,
"",
basedir=file_uri(input_basedir + "/"))
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
visit_class(job_order_object, ("File", "Directory"), pathToLoc)
adjustDirObjs(job_order_object, trim_listing)
normalizeFilesDirs(job_order_object)
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def load_job_order(args,
t,
parser,
stdin,
print_input_deps=False,
relative_deps=False,
stdout=sys.stdout):
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
"path": {
"@type": "@id"
},
"format": {
"@type": "@id"
},
"id": "@id"
}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {}
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(e, exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
_logger.debug("Parsed job order from command line: %s",
json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
parser.print_help()
if toolparser:
print "\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
if print_input_deps:
printdeps(job_order_object,
loader,
stdout,
relative_deps,
basedir="file://%s/" % input_basedir)
return 0
return (job_order_object, input_basedir)
def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout):
# type: (argparse.Namespace, Process, argparse.ArgumentParser, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]]
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
"path": {"@type": "@id"},
"format": {"@type": "@id"},
"id": "@id"}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file)
except Exception as e:
_logger.error(str(e), exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[str,str]
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(str(e), exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
parser.print_help()
if toolparser:
print u"\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required")
return 1
if print_input_deps:
printdeps(job_order_object, loader, stdout, relative_deps,
basedir=u"file://%s/" % input_basedir)
return 0
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def load_job_order(args,
t,
stdin,
print_input_deps=False,
relative_deps=False,
stdout=sys.stdout):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]]
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
u"path": {
u"@type": u"@id"
},
u"location": {
u"@type": u"@id"
},
u"format": {
u"@type": u"@id"
},
u"id": u"@id"
}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file,
checklinks=False)
except Exception as e:
_logger.error(str(e), exc_info=(e if args.debug else False))
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[unicode, unicode]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(
cmd_line["job_order"])
except Exception as e:
_logger.error(str(e),
exc_info=(e if args.debug else False))
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update(
{namemap[k]: v
for k, v in cmd_line.items()})
_logger.debug(u"Parsed job order from command line: %s",
json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"])
not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print u"\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object,
loader,
stdout,
relative_deps,
"",
basedir=u"file://%s/" % input_basedir)
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
adjustDirObjs(job_order_object, pathToLoc)
adjustFileObjs(job_order_object, pathToLoc)
normalizeFilesDirs(job_order_object)
adjustDirObjs(
job_order_object,
cast(Callable[..., Any],
functools.partial(getListing, StdFsAccess(input_basedir))))
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]]
job_order_object = None
_jobloaderctx = jobloaderctx.copy()
_jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.round_trip_load(stdin) # type: ignore
job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[Text, Text]
records = [] # type: List[Text]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), t, namemap, records)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
for record_name in records:
record = {}
record_items = {
k: v for k, v in cmd_line.iteritems()
if k.startswith(record_name)}
for key, value in record_items.iteritems():
record[key[len(record_name) + 1:]] = value
del cmd_line[key]
cmd_line[str(record_name)] = record
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(
os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
else:
job_order_object = {"id": args.workflow}
del cmd_line["job_order"]
job_order_object.update({namemap[k]: v for k, v in cmd_line.items()})
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print(u"\nOptions for {} ".format(args.workflow))
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object, loader, stdout, relative_deps, "",
basedir=file_uri(input_basedir + "/"))
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
visit_class(job_order_object, ("File", "Directory"), pathToLoc)
adjustDirObjs(job_order_object, trim_listing)
normalizeFilesDirs(job_order_object)
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def resolve_and_validate_document(
loadingContext, # type: LoadingContext
workflowobj, # type: Union[CommentedMap, CommentedSeq]
uri, # type: str
preprocess_only=False, # type: bool
skip_schemas=None, # type: Optional[bool]
):
# type: (...) -> Tuple[LoadingContext, str]
"""Validate a CWL document."""
if not loadingContext.loader:
raise ValueError("loadingContext must have a loader.")
else:
loader = loadingContext.loader
loadingContext = loadingContext.copy()
if not isinstance(workflowobj, MutableMapping):
raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
jobobj, _ = loader.resolve_all(workflowobj, uri)
uri = urllib.parse.urljoin(
uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"]
workflowobj = fetch_document(uri, loadingContext)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
cwlVersion = loadingContext.metadata.get("cwlVersion")
if not cwlVersion:
cwlVersion = workflowobj.get("cwlVersion")
if not cwlVersion and fileuri != uri:
# The tool we're loading is a fragment of a bigger file. Get
# the document root element and look for cwlVersion there.
metadata = fetch_document(fileuri,
loadingContext)[1] # type: Dict[str, Any]
cwlVersion = metadata.get("cwlVersion")
if not cwlVersion:
raise ValidationException(
"No cwlVersion found. "
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>.\n"
"Note: if this is a CWL draft-2 (pre v1.0) document then it "
"will need to be upgraded first.")
if not isinstance(cwlVersion, str):
with SourceLine(workflowobj, "cwlVersion", ValidationException):
raise ValidationException("'cwlVersion' must be a string, "
"got {}".format(type(cwlVersion)))
# strip out version
cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion)
if cwlVersion not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = []
for version in list(ALLUPDATES):
if "dev" in version:
version += " (with --enable-dev flag only)"
versions.append(version)
versions.sort()
raise ValidationException(
"The CWL reference runner no longer supports pre CWL v1.0 "
"documents. Supported versions are: "
"\n{}".format("\n".join(versions)))
if (isinstance(jobobj, CommentedMap)
and "http://commonwl.org/cwltool#overrides" in jobobj):
loadingContext.overrides_list.extend(
resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
if (isinstance(jobobj, CommentedMap)
and "https://w3id.org/cwl/cwl#requirements" in jobobj):
if cwlVersion not in ("v1.1.0-dev1", "v1.1"):
raise ValidationException(
"`cwl:requirements` in the input object is not part of CWL "
"v1.0. You can adjust to use `cwltool:overrides` instead; or you "
"can set the cwlVersion to v1.1 or greater.")
loadingContext.overrides_list.append({
"overrideTarget":
uri,
"requirements":
jobobj["https://w3id.org/cwl/cwl#requirements"],
})
del jobobj["https://w3id.org/cwl/cwl#requirements"]
(sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, str, None]
document_loader = Loader(
sch_document_loader.ctx,
schemagraph=sch_document_loader.graph,
idx=loader.idx,
cache=sch_document_loader.cache,
fetcher_constructor=loadingContext.fetcher_constructor,
skip_schemas=skip_schemas,
doc_cache=loadingContext.doc_cache,
)
if cwlVersion == "v1.0":
_add_blank_ids(workflowobj)
processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
if loadingContext.metadata:
metadata = loadingContext.metadata
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException(
"Workflow must be a CommentedMap or CommentedSeq.")
if not isinstance(metadata, CommentedMap):
raise ValidationException("metadata must be a CommentedMap, was %s" %
type(metadata))
if isinstance(processobj, CommentedMap):
uri = processobj["id"]
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return loadingContext, uri
if loadingContext.do_validate:
schema.validate_doc(avsc_names, processobj, document_loader,
loadingContext.strict)
# None means default behavior (do update)
if loadingContext.do_update in (True, None):
if "cwlVersion" not in metadata:
metadata["cwlVersion"] = cwlVersion
processobj = update.update(processobj, document_loader, fileuri,
loadingContext.enable_dev, metadata)
document_loader.idx[processobj["id"]] = processobj
def update_index(pr: CommentedMap) -> None:
if "id" in pr:
document_loader.idx[pr["id"]] = pr
visit_class(processobj,
("CommandLineTool", "Workflow", "ExpressionTool"),
update_index)
if isinstance(jobobj, CommentedMap):
loadingContext.jobdefaults = jobobj
loadingContext.loader = document_loader
loadingContext.avsc_names = avsc_names
loadingContext.metadata = metadata
return loadingContext, uri
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
stdout=sys.stdout, make_fs_access=None):
# type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Type[StdFsAccess]) -> Union[int, Tuple[Dict[Text, Any], Text]]
job_order_object = None
if args.conformance_test:
loader = Loader({})
else:
jobloaderctx = {
u"path": {u"@type": u"@id"},
u"location": {u"@type": u"@id"},
u"format": {u"@type": u"@id"},
u"id": u"@id"}
jobloaderctx.update(t.metadata.get("$namespaces", {}))
loader = Loader(jobloaderctx)
if len(args.job_order) == 1 and args.job_order[0][0] != "-":
job_order_file = args.job_order[0]
elif len(args.job_order) == 1 and args.job_order[0] == "-":
job_order_object = yaml.load(stdin)
job_order_object, _ = loader.resolve_all(job_order_object, "")
else:
job_order_file = None
if job_order_object:
input_basedir = args.basedir if args.basedir else os.getcwd()
elif job_order_file:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file))
try:
job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
toolparser = None
else:
input_basedir = args.basedir if args.basedir else os.getcwd()
namemap = {} # type: Dict[Text, Text]
toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
if toolparser:
if args.tool_help:
toolparser.print_help()
return 0
cmd_line = vars(toolparser.parse_args(args.job_order))
if cmd_line["job_order"]:
try:
input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"]))
job_order_object = loader.resolve_ref(cmd_line["job_order"])
except Exception as e:
_logger.error(Text(e), exc_info=args.debug)
return 1
else:
job_order_object = {"id": args.workflow}
job_order_object.update({namemap[k]: v for k,v in cmd_line.items()})
_logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
else:
job_order_object = None
for inp in t.tool["inputs"]:
if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object):
if not job_order_object:
job_order_object = {}
job_order_object[shortname(inp["id"])] = inp["default"]
if not job_order_object and len(t.tool["inputs"]) > 0:
if toolparser:
print u"\nOptions for %s " % args.workflow
toolparser.print_help()
_logger.error("")
_logger.error("Input object required, use --help for details")
return 1
if print_input_deps:
printdeps(job_order_object, loader, stdout, relative_deps, "",
basedir=u"file://%s/" % input_basedir)
return 0
def pathToLoc(p):
if "location" not in p and "path" in p:
p["location"] = p["path"]
del p["path"]
adjustDirObjs(job_order_object, pathToLoc)
adjustFileObjs(job_order_object, pathToLoc)
normalizeFilesDirs(job_order_object)
adjustDirObjs(job_order_object, cast(Callable[..., Any],
functools.partial(getListing, make_fs_access(input_basedir))))
if "cwl:tool" in job_order_object:
del job_order_object["cwl:tool"]
if "id" in job_order_object:
del job_order_object["id"]
return (job_order_object, input_basedir)
def validate_document(document_loader, # type: Loader
workflowobj, # type: CommentedMap
uri, # type: Text
enable_dev=False, # type: bool
strict=True, # type: bool
preprocess_only=False, # type: bool
fetcher_constructor=None, # type: FetcherConstructorType
skip_schemas=None, # type: bool
overrides=None, # type: List[Dict]
metadata=None, # type: Optional[Dict]
):
# type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
"""Validate a CWL document."""
if isinstance(workflowobj, list):
workflowobj = cmap({
"$graph": workflowobj
}, fn=uri)
if not isinstance(workflowobj, dict):
raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj))
jobobj = None
if "cwl:tool" in workflowobj:
job_loader = default_loader(fetcher_constructor) # type: ignore
jobobj, _ = job_loader.resolve_all(workflowobj, uri)
uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
if "http://commonwl.org/cwltool#overrides" in jobobj:
overrides.extend(resolve_overrides(jobobj, uri, uri))
del jobobj["http://commonwl.org/cwltool#overrides"]
workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]
fileuri = urllib.parse.urldefrag(uri)[0]
if "cwlVersion" not in workflowobj:
if metadata and 'cwlVersion' in metadata:
workflowobj['cwlVersion'] = metadata['cwlVersion']
else:
raise ValidationException("No cwlVersion found."
"Use the following syntax in your CWL document to declare "
"the version: cwlVersion: <version>")
if not isinstance(workflowobj["cwlVersion"], (str, Text)):
raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
# strip out version
workflowobj["cwlVersion"] = re.sub(
r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
workflowobj["cwlVersion"])
if workflowobj["cwlVersion"] not in list(ALLUPDATES):
# print out all the Supported Versions of cwlVersion
versions = list(ALLUPDATES) # ALLUPDATES is a dict
versions.sort()
raise ValidationException("'cwlVersion' not valid. Supported CWL versions are: \n{}".format("\n".join(versions)))
if workflowobj["cwlVersion"] == "draft-2":
workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1(
workflowobj, document_loader, uri, update_steps=False)))
if "@graph" in workflowobj:
workflowobj["$graph"] = workflowobj["@graph"]
del workflowobj["@graph"]
(sch_document_loader, avsc_names) = \
process.get_schema(workflowobj["cwlVersion"])[:2]
if isinstance(avsc_names, Exception):
raise avsc_names
processobj = None # type: Union[CommentedMap, CommentedSeq, Text]
document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
idx=document_loader.idx, cache=sch_document_loader.cache,
fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas)
_add_blank_ids(workflowobj)
workflowobj["id"] = fileuri
processobj, new_metadata = document_loader.resolve_all(workflowobj, fileuri)
if not isinstance(processobj, (CommentedMap, CommentedSeq)):
raise ValidationException("Workflow must be a dict or list.")
if not new_metadata:
if not isinstance(processobj, dict):
raise ValidationException("Draft-2 workflows must be a dict.")
new_metadata = cast(CommentedMap, cmap(
{"$namespaces": processobj.get("$namespaces", {}),
"$schemas": processobj.get("$schemas", []),
"cwlVersion": processobj["cwlVersion"]}, fn=fileuri))
_convert_stdstreams_to_files(workflowobj)
if preprocess_only:
return document_loader, avsc_names, processobj, new_metadata, uri
schema.validate_doc(avsc_names, processobj, document_loader, strict)
if new_metadata.get("cwlVersion") != update.LATEST:
processobj = cast(CommentedMap, cmap(update.update(
processobj, document_loader, fileuri, enable_dev, new_metadata)))
if jobobj:
new_metadata[u"cwl:defaults"] = jobobj
if overrides:
new_metadata[u"cwltool:overrides"] = overrides
return document_loader, avsc_names, processobj, new_metadata, uri