def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None): # type: (Dict[Text, Any], Loader, IO[Any], bool, Text, Text) -> None deps = {"class": "File", "location": uri} # type: Dict[Text, Any] def loadref(b, u): return document_loader.fetch(document_loader.fetcher.urljoin(b, u)) sf = scandeps( basedir if basedir else uri, obj, {"$import", "run"}, {"$include", "$schemas", "location"}, loadref) if sf: deps["secondaryFiles"] = sf if relative_deps: if relative_deps == "primary": base = basedir if basedir else os.path.dirname(uri_file_path(str(uri))) elif relative_deps == "cwd": base = os.getcwd() else: raise Exception(u"Unknown relative_deps %s" % relative_deps) visit_class(deps, ("File", "Directory"), functools.partial(makeRelative, base)) stdout.write(json.dumps(deps, indent=4))
def printdeps(obj, # type: Optional[Mapping[Text, Any]] document_loader, # type: Loader stdout, # type: Union[TextIO, StreamWriter] relative_deps, # type: bool uri, # type: Text prov_args=None, # type: Any basedir=None # type: Text ): # type: (...) -> Tuple[Optional[Dict[Text, Any]], Optional[Dict[Text, Any]]] """Print a JSON representation of the dependencies of the CWL document.""" deps = {"class": "File", "location": uri} # type: Dict[Text, Any] def loadref(base, uri): return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) sfs = scandeps( basedir if basedir else uri, obj, {"$import", "run"}, {"$include", "$schemas", "location"}, loadref) if sfs: deps["secondaryFiles"] = sfs if relative_deps: if relative_deps == "primary": base = basedir if basedir else os.path.dirname(uri_file_path(str(uri))) elif relative_deps == "cwd": base = os.getcwd() else: raise Exception(u"Unknown relative_deps %s" % relative_deps) absdeps = copy.deepcopy(deps) visit_class(deps, ("File", "Directory"), functools.partial(make_relative, base)) if prov_args: return (deps, absdeps) stdout.write(json_dumps(deps, indent=4)) return (None, None)
def visit(self, obj, stagedir, basedir, copy=False, staged=False): # type: (Dict[Text, Any], Text, Text, bool, bool) -> None tgt = os.path.join(stagedir, obj["basename"]) if obj["location"] in self._pathmap: return if obj["class"] == "Directory": if obj["location"].startswith("file://"): resolved = uri_file_path(obj["location"]) else: resolved = obj["location"] self._pathmap[obj["location"]] = MapperEnt(resolved, tgt, "WritableDirectory" if copy else "Directory", staged) if obj["location"].startswith("file://"): staged = False self.visitlisting(obj.get("listing", []), tgt, basedir, copy=copy, staged=staged) elif obj["class"] == "File": path = obj["location"] ab = abspath(path, basedir) if "contents" in obj and obj["location"].startswith("_:"): self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile", staged) else: with SourceLine(obj, "location", validate.ValidationException): # Dereference symbolic links deref = ab st = os.lstat(deref) while stat.S_ISLNK(st.st_mode): rl = os.readlink(deref) deref = rl if os.path.isabs(rl) else os.path.join( os.path.dirname(deref), rl) st = os.lstat(deref) self._pathmap[path] = MapperEnt(deref, tgt, "WritableFile" if copy else "File", staged) self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir, copy=copy, staged=staged)
def makeRelative(base, ob): u = ob.get("location", ob.get("path")) if ":" in u.split("/")[0] and not u.startswith("file://"): pass else: if u.startswith("file://"): u = uri_file_path(u) ob["location"] = os.path.relpath(u, base)
def make_relative(base, obj): """Relativize the location URI of a File or Directory object.""" uri = obj.get("location", obj.get("path")) if ":" in uri.split("/")[0] and not uri.startswith("file://"): pass else: if uri.startswith("file://"): uri = uri_file_path(uri) obj["location"] = os.path.relpath(uri, base)
def abspath(src, basedir): # type: (Text, Text) -> Text if src.startswith(u"file://"): ab = six.text_type(uri_file_path(str(src))) elif urllib.parse.urlsplit(src).scheme in ['http','https']: return src else: if basedir.startswith(u"file://"): ab = src if os.path.isabs(src) else basedir+ '/'+ src else: ab = src if os.path.isabs(src) else os.path.join(basedir, src) return ab
def revmap_file(builder, outdir, f): # type: (Builder, Text, Dict[Text, Any]) -> Union[Dict[Text, Any], None] """Remap a file from internal path to external path. For Docker, this maps from the path inside tho container to the path outside the container. Recognizes files in the pathmapper or remaps internal output directories to the external directory. """ split = urllib.parse.urlsplit(outdir) if not split.scheme: outdir = file_uri(str(outdir)) # builder.outdir is the inner (container/compute node) output directory # outdir is the outer (host/storage system) output directory if "location" in f and "path" not in f: if f["location"].startswith("file://"): f["path"] = convert_pathsep_to_unix(uri_file_path(f["location"])) else: return f if "path" in f: path = f["path"] uripath = file_uri(path) del f["path"] if "basename" not in f: f["basename"] = os.path.basename(path) assert builder.pathmapper is not None revmap_f = builder.pathmapper.reversemap(path) if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): f["location"] = revmap_f[1] elif uripath == outdir or uripath.startswith(outdir+os.sep): f["location"] = file_uri(path) elif path == builder.outdir or path.startswith(builder.outdir+os.sep): f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:]) elif not os.path.isabs(path): f["location"] = builder.fs_access.join(outdir, path) else: raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input " u"file pass through." % (path, builder.outdir)) return f raise WorkflowException(u"Output File object is missing both 'location' " "and 'path' fields: %s" % f)
def printdeps(obj, # type: Mapping[Text, Any] document_loader, # type: Loader stdout, # type: Union[TextIO, StreamWriter] relative_deps, # type: bool uri, # type: Text basedir=None, # type: Text nestdirs=True # type: bool ): # type: (...) -> None """Print a JSON representation of the dependencies of the CWL document.""" deps = find_deps(obj, document_loader, uri, basedir=basedir, nestdirs=nestdirs) if relative_deps == "primary": base = basedir if basedir else os.path.dirname(uri_file_path(str(uri))) elif relative_deps == "cwd": base = os.getcwd() visit_class(deps, ("File", "Directory"), functools.partial( make_relative, base)) stdout.write(json_dumps(deps, indent=4))
def revmap_file(builder, outdir, f): # type: (Builder, Text, Dict[Text, Any]) -> Union[Dict[Text, Any], None] """Remap a file from internal path to external path. For Docker, this maps from the path inside tho container to the path outside the container. Recognizes files in the pathmapper or remaps internal output directories to the external directory. """ split = urllib.parse.urlsplit(outdir) if not split.scheme: outdir = file_uri(str(outdir)) if "location" in f: if f["location"].startswith("file://"): path = uri_file_path(f["location"]) revmap_f = builder.pathmapper.reversemap(path) if revmap_f: f["location"] = revmap_f[1] elif path == builder.outdir: f["location"] = outdir elif path.startswith(builder.outdir): f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:]) return f if "path" in f: path = f["path"] del f["path"] revmap_f = builder.pathmapper.reversemap(path) if revmap_f: f["location"] = revmap_f[1] return f elif path.startswith(builder.outdir): f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:]) return f else: raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input " u"file pass through." % (path, builder.outdir)) raise WorkflowException(u"Output File object is missing both `location` and `path` fields: %s" % f)
def main(args=None): parser = argparse.ArgumentParser() parser.add_argument("workflow_file", type=str, help="CWL workflow specification file") parser.add_argument("job_file", nargs='+', help="One or more whitespace separated job files") # directory to write any outputs to # defaults to the current working directory parser.add_argument("--outdir", type=str, default=os.getcwd(), help="Directory to write the outputs to. "\ "Defaults to the current working directory.") # if executed from command line, update args to those if args is None: args = sys.argv[1:] options = parser.parse_args(args) print("Options: " + str(options)) # create a cwltool object from the cwl workflow file try: tool = cwltool.load_tool.load_tool( options.workflow_file, makeDataCommonsTool, kwargs={}, resolver=cwltool.resolver.tool_resolver ) print("Tool:") print(vars(tool)) except cwltool.process.UnsupportedRequirement as e: print("UnsupportedRequirement") # set up args for load_job_order options.workflow = options.workflow_file options.job_order = options.job_file # set default basedir to be the cwd. #Maybe set to None and let load_job_order determine it options.basedir = os.getcwd() options.tool_help = None options.debug = True # delayed import cwltool.main (circular dependency) import cwltool.main # load the job files job, _ = cwltool.main.load_job_order(options, tool, sys.stdin) print("Job: ") pprint(job) for inputname in job: print("inputname: {}".format(inputname)) if inputname == "file": filearg = job["file"] print("filearg: {}".format(filearg)) if filearg.get("location"): filearg["path"] = uri_file_path(filearg["location"]) kwargs = { 'basedir': options.basedir, 'outdir': options.basedir } jobiter = tool.job(job, None, **kwargs) for jobrunner in jobiter: if jobrunner: jobrunner.run(**kwargs) else: print("")
def locToPath(p): if "path" not in p and "location" in p: p["path"] = uri_file_path(p["location"])
def loc_to_path(obj): for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"])
def visit( self, obj: CWLObjectType, stagedir: str, basedir: str, copy: bool = False, staged: bool = False, ) -> None: tgt = convert_pathsep_to_unix( os.path.join(stagedir, cast(str, obj["basename"])) ) if obj["location"] in self._pathmap: return if obj["class"] == "Directory": location = cast(str, obj["location"]) if location.startswith("file://"): resolved = uri_file_path(location) else: resolved = location self._pathmap[location] = MapperEnt( resolved, tgt, "WritableDirectory" if copy else "Directory", staged ) if location.startswith("file://"): staged = False self.visitlisting( cast(List[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, staged=staged, ) elif obj["class"] == "File": path = cast(str, obj["location"]) ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( obj["contents"], tgt, "CreateWritableFile" if copy else "CreateFile", staged, ) else: with SourceLine( obj, "location", ValidationException, _logger.isEnabledFor(logging.DEBUG), ): deref = ab if urllib.parse.urlsplit(deref).scheme in ["http", "https"]: deref = downloadHttpFile(path) else: # Dereference symbolic links st = os.lstat(deref) while stat.S_ISLNK(st.st_mode): rl = os.readlink(deref) deref = ( rl if os.path.isabs(rl) else os.path.join(os.path.dirname(deref), rl) ) st = os.lstat(deref) self._pathmap[path] = MapperEnt( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( cast(List[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, )
def revmap_file(builder, outdir, f): # type: (Builder, str, Dict[str, Any]) -> Union[Dict[str, Any], None] """ Remap a file from internal path to external path. For Docker, this maps from the path inside tho container to the path outside the container. Recognizes files in the pathmapper or remaps internal output directories to the external directory. """ split = urllib.parse.urlsplit(outdir) if not split.scheme: outdir = file_uri(str(outdir)) # builder.outdir is the inner (container/compute node) output directory # outdir is the outer (host/storage system) output directory if "location" in f and "path" not in f: if f["location"].startswith("file://"): f["path"] = convert_pathsep_to_unix(uri_file_path(f["location"])) else: return f if "path" in f: path = f["path"] uripath = file_uri(path) del f["path"] if "basename" not in f: f["basename"] = os.path.basename(path) if not builder.pathmapper: raise ValueError( "Do not call revmap_file using a builder that doesn't have a pathmapper." ) revmap_f = builder.pathmapper.reversemap(path) if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith( "Writable" ): f["location"] = revmap_f[1] elif ( uripath == outdir or uripath.startswith(outdir + os.sep) or uripath.startswith(outdir + "/") ): f["location"] = file_uri(path) elif ( path == builder.outdir or path.startswith(builder.outdir + os.sep) or path.startswith(builder.outdir + "/") ): f["location"] = builder.fs_access.join( outdir, path[len(builder.outdir) + 1 :] ) elif not os.path.isabs(path): f["location"] = builder.fs_access.join(outdir, path) else: raise WorkflowException( "Output file path %s must be within designated output directory (%s) or an input " "file pass through." % (path, builder.outdir) ) return f raise WorkflowException( "Output File object is missing both 'location' " "and 'path' fields: %s" % f )
def locToPath(p): for field in ("path", "nameext", "nameroot", "dirname"): if field in p: del p[field] if p["location"].startswith("file://"): p["path"] = uri_file_path(p["location"])
def _realpath(ob): # type: (Dict[Text, Any]) -> None if ob["location"].startswith("file:"): ob["location"] = file_uri( os.path.realpath(uri_file_path(ob["location"]))) if ob["location"].startswith("/"): ob["location"] = os.path.realpath(ob["location"])
def abspath(src, basedir): # type: (Text, Text) -> Text if src.startswith(u"file://"): ab = unicode(uri_file_path(str(src))) else: ab = src if os.path.isabs(src) else os.path.join(basedir, src) return ab
def loc_to_path(obj): # type: (Dict[str, Any]) -> None for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"])
def locToPath(p): if p["location"].startswith("file://"): p["path"] = uri_file_path(p["location"])
def loc_to_path(obj: CWLObjectType) -> None: for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if cast(str, obj["location"]).startswith("file://"): obj["path"] = uri_file_path(cast(str, obj["location"]))