def load_job_order(args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Dict[Text, Any], Text, Loader] job_order_object = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object: overrides.extend(resolve_overrides(job_order_object, file_uri(job_order_file), tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if not job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() return (job_order_object, input_basedir, loader)
def test_attachments(): furi = file_uri(get_data("schema_salad/tests/multidoc.yml")) l1 = Loader({}) r1, _ = l1.resolve_ref(furi) with open(get_data("schema_salad/tests/multidoc.yml"), "rt") as f: content = f.read() assert {"foo": "bar", "baz": content, "quux": content} == r1 l2 = Loader({}, allow_attachments=lambda x: x["foo"] == "bar") r2, _ = l2.resolve_ref(furi) assert { "foo": "bar", "baz": "This is the {first attachment}.\n", "quux": "This is the [second attachment].", } == r2 l3 = Loader({}, allow_attachments=lambda x: x["foo"] == "baz") r3, _ = l3.resolve_ref(furi) with open(get_data("schema_salad/tests/multidoc.yml"), "rt") as f: content = f.read() assert {"foo": "bar", "baz": content, "quux": content} == r3
def test_fetch_inject_id() -> None: path = get_data("schema_salad/tests/inject-id1.yml") assert path if is_fs_case_sensitive(os.path.dirname(path)): def lower(item: str) -> str: return item else: def lower(item: str) -> str: return item.lower() l1 = Loader({"id": "@id"}) furi1 = file_uri(path) r1, _ = l1.resolve_ref(furi1) assert {"id": furi1 + "#foo", "bar": "baz"} == r1 assert [lower(furi1), lower(furi1 + "#foo")] == sorted( list(lower(k) for k in l1.idx.keys()) ) l2 = Loader({"id": "@id"}) path2 = get_data("schema_salad/tests/inject-id2.yml") assert path2 furi2 = file_uri(path2) r2, _ = l2.resolve_ref(furi2) assert {"id": furi2, "bar": "baz"} == r2 assert [lower(furi2)] == sorted(list(lower(k) for k in l2.idx.keys())) l3 = Loader({"id": "@id"}) path3 = get_data("schema_salad/tests/inject-id3.yml") assert path3 furi3 = file_uri(path3) r3, _ = l3.resolve_ref(furi3) assert {"id": "http://example.com", "bar": "baz"} == r3 assert [lower(furi3), "http://example.com"] == sorted( list(lower(k) for k in l3.idx.keys()) )
def load_job_order( args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides_list, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader] job_order_object = None job_order_file = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object is not None: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file is not None: input_basedir = args.basedir if args.basedir \ else os.path.abspath(os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object: ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend( resolve_overrides(job_order_object, ov_uri, tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if job_order_object is None: input_basedir = args.basedir if args.basedir else os.getcwd() if job_order_object is not None and not isinstance(job_order_object, MutableMapping): _logger.error( 'CWL input object at %s is not formatted correctly, it should be a ' 'JSON/YAML dictionay, not %s.\n' 'Raw input object:\n%s', job_order_file or "stdin", type(job_order_object), job_order_object) sys.exit(1) return (job_order_object, input_basedir, loader)
def test_fetch_inject_id(): lower = lambda s: s.lower() if is_fs_case_sensitive( os.path.dirname(get_data("schema_salad/tests/inject-id1.yml"))): lower = lambda a: a l1 = Loader({"id": "@id"}) furi1 = file_uri(get_data("schema_salad/tests/inject-id1.yml")) r1, _ = l1.resolve_ref(furi1) assert {"id": furi1 + "#foo", "bar": "baz"} == r1 assert [lower(furi1), lower(furi1 + "#foo") ] == sorted(list(lower(k) for k in l1.idx.keys())) l2 = Loader({"id": "@id"}) furi2 = file_uri(get_data("schema_salad/tests/inject-id2.yml")) r2, _ = l2.resolve_ref(furi2) assert {"id": furi2, "bar": "baz"} == r2 assert [lower(furi2)] == sorted(list(lower(k) for k in l2.idx.keys())) l3 = Loader({"id": "@id"}) furi3 = file_uri(get_data("schema_salad/tests/inject-id3.yml")) r3, _ = l3.resolve_ref(furi3) assert {"id": "http://example.com", "bar": "baz"} == r3 assert [lower(furi3), "http://example.com" ] == sorted(list(lower(k) for k in l3.idx.keys()))
def test_sourceline() -> None: ldr = Loader({"id": "@id"}) path = get_data("tests/frag.yml") assert path b, _ = ldr.resolve_ref(path) class TestExp(Exception): pass try: with SourceLine(b, 1, TestExp, False): raise Exception("Whoops") except TestExp as e: assert str(e).endswith("frag.yml:3:3: Whoops"), e except Exception as exc: assert False, exc
def test_mixin() -> None: base_url = file_uri(os.path.join(os.getcwd(), "tests")) ldr = Loader({}) path = get_data("tests/mixin.yml") assert path ra = ldr.resolve_ref(cmap({"$mixin": path, "one": "five"}), base_url=base_url) assert {"id": "four", "one": "five"} == ra[0] ldr = Loader({"id": "@id"}) ra = ldr.resolve_all( cmap([{"id": "a", "m": {"$mixin": path}}, {"id": "b", "m": {"$mixin": path}}]), base_url=base_url, ) assert [ {"id": base_url + "#a", "m": {"id": base_url + "#a/four", "one": "two"}}, {"id": base_url + "#b", "m": {"id": base_url + "#b/four", "one": "two"}}, ] == ra[0]
def load_job_order(args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides_list, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader] job_order_object = None job_order_file = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object is not None: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file is not None: input_basedir = args.basedir if args.basedir \ else os.path.abspath(os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object: ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend( resolve_overrides(job_order_object, ov_uri, tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if job_order_object is None: input_basedir = args.basedir if args.basedir else os.getcwd() if job_order_object is not None and not isinstance(job_order_object, MutableMapping): _logger.error( 'CWL input object at %s is not formatted correctly, it should be a ' 'JSON/YAML dictionay, not %s.\n' 'Raw input object:\n%s', job_order_file or "stdin", type(job_order_object), job_order_object) sys.exit(1) return (job_order_object, input_basedir, loader)
def load_job_order( args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Dict[Text, Any], Text, Loader] job_order_object = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object: overrides.extend( resolve_overrides(job_order_object, file_uri(job_order_file), tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if not job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() return (job_order_object, input_basedir, loader)
def resolve_and_validate_document( loadingContext, # type: LoadingContext workflowobj, # type: Union[CommentedMap, CommentedSeq] uri, # type: str preprocess_only=False, # type: bool skip_schemas=None, # type: Optional[bool] ): # type: (...) -> Tuple[LoadingContext, str] """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") else: loader = loadingContext.loader loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = loadingContext.metadata.get("cwlVersion") if not cwlVersion: cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion and fileuri != uri: # The tool we're loading is a fragment of a bigger file. Get # the document root element and look for cwlVersion there. metadata = fetch_document(fileuri, loadingContext)[1] # type: Dict[str, Any] cwlVersion = metadata.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, str): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format(type(cwlVersion))) # strip out version cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if (isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj): loadingContext.overrides_list.extend( resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if (isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj): if cwlVersion not in ("v1.1.0-dev1", "v1.1"): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1 or greater.") loadingContext.overrides_list.append({ "overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], }) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, str, None] document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas, doc_cache=loadingContext.doc_cache, ) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) document_loader.resolve_all(workflowobj, fileuri) processobj, metadata = document_loader.resolve_ref(uri) if loadingContext.metadata: metadata = loadingContext.metadata if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException( "Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) if isinstance(processobj, CommentedMap): uri = processobj["id"] _convert_stdstreams_to_files(workflowobj) if isinstance(jobobj, CommentedMap): loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) # None means default behavior (do update) if loadingContext.do_update in (True, None): if "cwlVersion" not in metadata: metadata["cwlVersion"] = cwlVersion processobj = update.update(processobj, document_loader, fileuri, loadingContext.enable_dev, metadata) document_loader.idx[processobj["id"]] = processobj def update_index(pr: CommentedMap) -> None: if "id" in pr: document_loader.idx[pr["id"]] = pr visit_class(processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index) return loadingContext, uri
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[Text, Text], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None _jobloaderctx = jobloaderctx.copy() _jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] records = [] # type: List[Text] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap, records) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) for record_name in records: record = {} record_items = { k: v for k, v in six.iteritems(cmd_line) if k.startswith(record_name) } for key, value in six.iteritems(record_items): record[key[len(record_name) + 1:]] = value del cmd_line[key] cmd_line[str(record_name)] = record if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref( cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} del cmd_line["job_order"] job_order_object.update( {namemap[k]: v for k, v in cmd_line.items()}) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print(u"\nOptions for {} ".format(args.workflow)) toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=file_uri(input_basedir + "/")) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] visit_class(job_order_object, ("File", "Directory"), pathToLoc) adjustDirObjs(job_order_object, trim_listing) normalizeFilesDirs(job_order_object) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout): # type: (argparse.Namespace, Process, argparse.ArgumentParser, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]] job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { "path": {"@type": "@id"}, "format": {"@type": "@id"}, "id": "@id"} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[str,str] toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: parser.print_help() if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, basedir=u"file://%s/" % input_basedir) return 0 if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, parser=None): if args is None: args = sys.argv[1:] if parser is None: parser = arg_parser() args = parser.parse_args(args) if args.quiet: _logger.setLevel(logging.WARN) if args.debug: _logger.setLevel(logging.DEBUG) pkg = pkg_resources.require("cwltool") if pkg: if args.version: print "%s %s" % (sys.argv[0], pkg[0].version) return 0 else: _logger.info("%s %s", sys.argv[0], pkg[0].version) if not args.workflow: parser.print_help() _logger.error("") _logger.error("CWL document required") return 1 t = load_tool(args.workflow, args.update, args.strict, makeTool, args.debug) if type(t) == int: return t if args.print_rdf: printrdf(args.workflow, processobj, ctx, args.rdf_serializer) return 0 if args.print_dot: printdot(args.workflow, processobj, ctx, args.rdf_serializer) return 0 if args.tmp_outdir_prefix != 'tmp': # Use user defined temp directory (if it exists) args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix) if not os.path.exists(args.tmp_outdir_prefix): _logger.error("Intermediate output directory prefix doesn't exist, reverting to default") return 1 if args.tmpdir_prefix != 'tmp': # Use user defined prefix (if the folder exists) args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix) if not os.path.exists(args.tmpdir_prefix): _logger.error("Temporary directory prefix doesn't exist.") return 1 if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] else: job_order_file = None if args.conformance_test: loader = Loader({}) else: loader = Loader({"id": "@id", "path": {"@type": "@id"}}) if job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 else: job_order_object = {} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", job_order_object) else: job_order_object = None if not job_order_object: parser.print_help() if toolparser: print "\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 try: out = executor(t, job_order_object, input_basedir, args, conformance_test=args.conformance_test, dry_run=args.dry_run, outdir=args.outdir, tmp_outdir_prefix=args.tmp_outdir_prefix, use_container=args.use_container, preserve_environment=args.preserve_environment, pull_image=args.enable_pull, rm_container=args.rm_container, tmpdir_prefix=args.tmpdir_prefix, rm_tmpdir=args.rm_tmpdir, makeTool=makeTool, move_outputs=args.move_outputs ) # This is the workflow output, it needs to be written sys.stdout.write(json.dumps(out, indent=4)) except (validate.ValidationException) as e: _logger.error("Input object failed validation:\n%s", e, exc_info=(e if args.debug else False)) return 1 except workflow.WorkflowException as e: _logger.error("Workflow error:\n %s", e, exc_info=(e if args.debug else False)) return 1 return 0
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None _jobloaderctx = jobloaderctx.copy() _jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) # type: ignore job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] records = [] # type: List[Text] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap, records) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) for record_name in records: record = {} record_items = { k: v for k, v in cmd_line.iteritems() if k.startswith(record_name)} for key, value in record_items.iteritems(): record[key[len(record_name) + 1:]] = value del cmd_line[key] cmd_line[str(record_name)] = record if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} del cmd_line["job_order"] job_order_object.update({namemap[k]: v for k, v in cmd_line.items()}) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print(u"\nOptions for {} ".format(args.workflow)) toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=file_uri(input_basedir + "/")) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] visit_class(job_order_object, ("File", "Directory"), pathToLoc) adjustDirObjs(job_order_object, trim_listing) normalizeFilesDirs(job_order_object) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]] job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { u"path": { u"@type": u"@id" }, u"location": { u"@type": u"@id" }, u"format": { u"@type": u"@id" }, u"id": u"@id" } jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[unicode, unicode] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref( cmd_line["job_order"]) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update( {namemap[k]: v for k, v in cmd_line.items()}) _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=u"file://%s/" % input_basedir) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] adjustDirObjs(job_order_object, pathToLoc) adjustFileObjs(job_order_object, pathToLoc) normalizeFilesDirs(job_order_object) adjustDirObjs( job_order_object, cast(Callable[..., Any], functools.partial(getListing, StdFsAccess(input_basedir)))) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout): job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { "path": { "@type": "@id" }, "format": { "@type": "@id" }, "id": "@id" } jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref( cmd_line["job_order"]) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update( {namemap[k]: v for k, v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: parser.print_help() if toolparser: print "\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, basedir="file://%s/" % input_basedir) return 0 return (job_order_object, input_basedir)
def load_job_order(args, t, parser): job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 else: job_order_object = {} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", job_order_object) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: parser.print_help() if toolparser: print "\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 return (job_order_object, input_basedir)
def load_job(job_file): loader = Loader(load.jobloaderctx.copy()) job_order_object, _ = loader.resolve_ref(job_file, checklinks=False) return job_order_object
def init_job_order( job_order_object: Optional[CWLObjectType], args: argparse.Namespace, process: Process, loader: Loader, stdout: Union[TextIO, StreamWriter], print_input_deps: bool = False, relative_deps: str = "primary", make_fs_access: Callable[[str], StdFsAccess] = StdFsAccess, input_basedir: str = "", secret_store: Optional[SecretStore] = None, input_required: bool = True, ) -> CWLObjectType: secrets_req, _ = process.get_requirement( "http://commonwl.org/cwltool#Secrets") if job_order_object is None: namemap = {} # type: Dict[str, str] records = [] # type: List[str] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), process, namemap, records, input_required, ) if args.tool_help: toolparser.print_help() exit(0) cmd_line = vars(toolparser.parse_args(args.job_order)) for record_name in records: record = {} record_items = { k: v for k, v in cmd_line.items() if k.startswith(record_name) } for key, value in record_items.items(): record[key[len(record_name) + 1:]] = value del cmd_line[key] cmd_line[str(record_name)] = record if "job_order" in cmd_line and cmd_line["job_order"]: try: job_order_object = cast( CWLObjectType, loader.resolve_ref(cmd_line["job_order"])[0], ) except Exception: _logger.exception("Failed to resolv job_order: %s", cmd_line["job_order"]) exit(1) else: job_order_object = {"id": args.workflow} del cmd_line["job_order"] job_order_object.update({namemap[k]: v for k, v in cmd_line.items()}) if secret_store and secrets_req: secret_store.store( [ shortname(sc) for sc in cast(List[str], secrets_req["secrets"]) ], job_order_object, ) if _logger.isEnabledFor(logging.DEBUG): _logger.debug( "Parsed job order from command line: %s", json_dumps(job_order_object, indent=4), ) for inp in process.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if job_order_object is None: if process.tool["inputs"]: if toolparser is not None: print("\nOptions for {} ".format(args.workflow)) toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") exit(1) else: job_order_object = {} if print_input_deps: basedir = None # type: Optional[str] uri = cast(str, job_order_object["id"]) if uri == args.workflow: basedir = os.path.dirname(uri) uri = "" printdeps( job_order_object, loader, stdout, relative_deps, uri, basedir=basedir, nestdirs=False, ) exit(0) def path_to_loc(p: CWLObjectType) -> None: if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] ns = {} # type: ContextType ns.update(cast(ContextType, job_order_object.get("$namespaces", {}))) ns.update(cast(ContextType, process.metadata.get("$namespaces", {}))) ld = Loader(ns) def expand_formats(p: CWLObjectType) -> None: if "format" in p: p["format"] = ld.expand_url(cast(str, p["format"]), "") visit_class(job_order_object, ("File", "Directory"), path_to_loc) visit_class( job_order_object, ("File", ), functools.partial(add_sizes, make_fs_access(input_basedir)), ) visit_class(job_order_object, ("File", ), expand_formats) adjustDirObjs(job_order_object, trim_listing) normalizeFilesDirs(job_order_object) if secret_store and secrets_req: secret_store.store( [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], job_order_object, ) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return job_order_object
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Type[StdFsAccess]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { u"path": {u"@type": u"@id"}, u"location": {u"@type": u"@id"}, u"format": {u"@type": u"@id"}, u"id": u"@id"} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=u"file://%s/" % input_basedir) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] adjustDirObjs(job_order_object, pathToLoc) adjustFileObjs(job_order_object, pathToLoc) normalizeFilesDirs(job_order_object) adjustDirObjs(job_order_object, cast(Callable[..., Any], functools.partial(getListing, make_fs_access(input_basedir)))) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def pack( document_loader: Loader, processobj, # type: Union[Dict[str, Any], List[Dict[str, Any]]] uri, # type: str metadata, # type: Dict[str, str] rewrite_out=None, # type: Optional[Dict[str, str]] ): # type: (...) -> Dict[str, Any] document_loader = SubLoader(document_loader) document_loader.idx = {} if isinstance(processobj, MutableMapping): document_loader.idx[processobj["id"]] = CommentedMap( processobj.items()) elif isinstance(processobj, MutableSequence): _, frag = urllib.parse.urldefrag(uri) for po in processobj: if not frag: if po["id"].endswith("#main"): uri = po["id"] document_loader.idx[po["id"]] = CommentedMap(po.items()) document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) def loadref(base, uri): # type: (Optional[str], str) -> Union[Dict[str, Any], List[Dict[str, Any]], str, None] return document_loader.resolve_ref(uri, base_url=base)[0] ids = set() # type: Set[str] find_ids(processobj, ids) runs = {uri} find_run(processobj, loadref, runs) for f in runs: find_ids(document_loader.resolve_ref(f)[0], ids) names = set() # type: Set[str] if rewrite_out is None: rewrite = {} # type: Dict[str, str] else: rewrite = rewrite_out mainpath, _ = urllib.parse.urldefrag(uri) def rewrite_id(r, mainuri): # type: (str, str) -> None if r == mainuri: rewrite[r] = "#main" elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): if r[len(mainuri):].startswith("#main/"): rewrite[r] = "#" + uniquename(r[len(mainuri) + 1:], names) else: rewrite[r] = "#" + uniquename("main/" + r[len(mainuri) + 1:], names) else: path, frag = urllib.parse.urldefrag(r) if path == mainpath: rewrite[r] = "#" + uniquename(frag, names) else: if path not in rewrite: rewrite[path] = "#" + uniquename(shortname(path), names) sortedids = sorted(ids) for r in sortedids: rewrite_id(r, uri) packed = CommentedMap( (("$graph", CommentedSeq()), ("cwlVersion", metadata["cwlVersion"]))) namespaces = metadata.get("$namespaces", None) schemas = set() # type: Set[str] if "$schemas" in metadata: for each_schema in metadata["$schemas"]: schemas.add(each_schema) for r in sorted(runs): dcr, metadata = document_loader.resolve_ref(r) if isinstance(dcr, CommentedSeq): dcr = dcr[0] dcr = cast(CommentedMap, dcr) if not isinstance(dcr, MutableMapping): continue metadata = cast(Dict[str, Any], metadata) if "$schemas" in metadata: for s in metadata["$schemas"]: schemas.add(s) if dcr.get("class") not in ("Workflow", "CommandLineTool", "ExpressionTool"): continue dc = cast(Dict[str, Any], copy.deepcopy(dcr)) v = rewrite[r] dc["id"] = v for n in ("name", "cwlVersion", "$namespaces", "$schemas"): if n in dc: del dc[n] packed["$graph"].append(dc) if schemas: packed["$schemas"] = list(schemas) for r in list(rewrite.keys()): v = rewrite[r] replace_refs(packed, rewrite, r + "/" if "#" in r else r + "#", v + "/") import_embed(packed, set()) if len(packed["$graph"]) == 1: # duplicate 'cwlVersion' and $schemas inside $graph when there is only # a single item because we will print the contents inside '$graph' # rather than whole dict packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"] if schemas: packed["$graph"][0]["$schemas"] = list(schemas) # always include $namespaces in the #main if namespaces: packed["$graph"][0]["$namespaces"] = namespaces return packed