def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None):
# type: (Dict[Text, Any], Loader, IO[Any], bool, Text, Text) -> None
deps = {"class": "File",
"location": uri} # type: Dict[Text, Any]
def loadref(b, u):
return document_loader.fetch(document_loader.fetcher.urljoin(b, u))
sf = scandeps(
basedir if basedir else uri, obj, {"$import", "run"},
{"$include", "$schemas", "location"}, loadref)
if sf:
deps["secondaryFiles"] = sf
if relative_deps:
if relative_deps == "primary":
base = basedir if basedir else os.path.dirname(uri_file_path(str(uri)))
elif relative_deps == "cwd":
base = os.getcwd()
else:
raise Exception(u"Unknown relative_deps %s" % relative_deps)
visit_class(deps, ("File", "Directory"), functools.partial(makeRelative, base))
stdout.write(json.dumps(deps, indent=4))
def printdeps(obj, # type: Optional[Mapping[Text, Any]]
document_loader, # type: Loader
stdout, # type: Union[TextIO, StreamWriter]
relative_deps, # type: bool
uri, # type: Text
prov_args=None, # type: Any
basedir=None # type: Text
): # type: (...) -> Tuple[Optional[Dict[Text, Any]], Optional[Dict[Text, Any]]]
"""Print a JSON representation of the dependencies of the CWL document."""
deps = {"class": "File", "location": uri} # type: Dict[Text, Any]
def loadref(base, uri):
return document_loader.fetch(document_loader.fetcher.urljoin(base, uri))
sfs = scandeps(
basedir if basedir else uri, obj, {"$import", "run"},
{"$include", "$schemas", "location"}, loadref)
if sfs:
deps["secondaryFiles"] = sfs
if relative_deps:
if relative_deps == "primary":
base = basedir if basedir else os.path.dirname(uri_file_path(str(uri)))
elif relative_deps == "cwd":
base = os.getcwd()
else:
raise Exception(u"Unknown relative_deps %s" % relative_deps)
absdeps = copy.deepcopy(deps)
visit_class(deps, ("File", "Directory"), functools.partial(make_relative, base))
if prov_args:
return (deps, absdeps)
stdout.write(json_dumps(deps, indent=4))
return (None, None)
def visit(self, obj, stagedir, basedir, copy=False, staged=False):
# type: (Dict[Text, Any], Text, Text, bool, bool) -> None
tgt = os.path.join(stagedir, obj["basename"])
if obj["location"] in self._pathmap:
return
if obj["class"] == "Directory":
if obj["location"].startswith("file://"):
resolved = uri_file_path(obj["location"])
else:
resolved = obj["location"]
self._pathmap[obj["location"]] = MapperEnt(resolved, tgt, "WritableDirectory" if copy else "Directory", staged)
if obj["location"].startswith("file://"):
staged = False
self.visitlisting(obj.get("listing", []), tgt, basedir, copy=copy, staged=staged)
elif obj["class"] == "File":
path = obj["location"]
ab = abspath(path, basedir)
if "contents" in obj and obj["location"].startswith("_:"):
self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile", staged)
else:
with SourceLine(obj, "location", validate.ValidationException):
# Dereference symbolic links
deref = ab
st = os.lstat(deref)
while stat.S_ISLNK(st.st_mode):
rl = os.readlink(deref)
deref = rl if os.path.isabs(rl) else os.path.join(
os.path.dirname(deref), rl)
st = os.lstat(deref)
self._pathmap[path] = MapperEnt(deref, tgt, "WritableFile" if copy else "File", staged)
self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir, copy=copy, staged=staged)
def makeRelative(base, ob):
u = ob.get("location", ob.get("path"))
if ":" in u.split("/")[0] and not u.startswith("file://"):
pass
else:
if u.startswith("file://"):
u = uri_file_path(u)
ob["location"] = os.path.relpath(u, base)
def make_relative(base, obj):
"""Relativize the location URI of a File or Directory object."""
uri = obj.get("location", obj.get("path"))
if ":" in uri.split("/")[0] and not uri.startswith("file://"):
pass
else:
if uri.startswith("file://"):
uri = uri_file_path(uri)
obj["location"] = os.path.relpath(uri, base)
def abspath(src, basedir): # type: (Text, Text) -> Text
if src.startswith(u"file://"):
ab = six.text_type(uri_file_path(str(src)))
elif urllib.parse.urlsplit(src).scheme in ['http','https']:
return src
else:
if basedir.startswith(u"file://"):
ab = src if os.path.isabs(src) else basedir+ '/'+ src
else:
ab = src if os.path.isabs(src) else os.path.join(basedir, src)
return ab
def revmap_file(builder, outdir, f):
# type: (Builder, Text, Dict[Text, Any]) -> Union[Dict[Text, Any], None]
"""Remap a file from internal path to external path.
For Docker, this maps from the path inside tho container to the path
outside the container. Recognizes files in the pathmapper or remaps
internal output directories to the external directory.
"""
split = urllib.parse.urlsplit(outdir)
if not split.scheme:
outdir = file_uri(str(outdir))
# builder.outdir is the inner (container/compute node) output directory
# outdir is the outer (host/storage system) output directory
if "location" in f and "path" not in f:
if f["location"].startswith("file://"):
f["path"] = convert_pathsep_to_unix(uri_file_path(f["location"]))
else:
return f
if "path" in f:
path = f["path"]
uripath = file_uri(path)
del f["path"]
if "basename" not in f:
f["basename"] = os.path.basename(path)
assert builder.pathmapper is not None
revmap_f = builder.pathmapper.reversemap(path)
if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"):
f["location"] = revmap_f[1]
elif uripath == outdir or uripath.startswith(outdir+os.sep):
f["location"] = file_uri(path)
elif path == builder.outdir or path.startswith(builder.outdir+os.sep):
f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:])
elif not os.path.isabs(path):
f["location"] = builder.fs_access.join(outdir, path)
else:
raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input "
u"file pass through." % (path, builder.outdir))
return f
raise WorkflowException(u"Output File object is missing both 'location' "
"and 'path' fields: %s" % f)
def printdeps(obj, # type: Mapping[Text, Any]
document_loader, # type: Loader
stdout, # type: Union[TextIO, StreamWriter]
relative_deps, # type: bool
uri, # type: Text
basedir=None, # type: Text
nestdirs=True # type: bool
): # type: (...) -> None
"""Print a JSON representation of the dependencies of the CWL document."""
deps = find_deps(obj, document_loader, uri, basedir=basedir,
nestdirs=nestdirs)
if relative_deps == "primary":
base = basedir if basedir else os.path.dirname(uri_file_path(str(uri)))
elif relative_deps == "cwd":
base = os.getcwd()
visit_class(deps, ("File", "Directory"), functools.partial(
make_relative, base))
stdout.write(json_dumps(deps, indent=4))
def revmap_file(builder, outdir, f):
# type: (Builder, Text, Dict[Text, Any]) -> Union[Dict[Text, Any], None]
"""Remap a file from internal path to external path.
For Docker, this maps from the path inside tho container to the path
outside the container. Recognizes files in the pathmapper or remaps
internal output directories to the external directory.
"""
split = urllib.parse.urlsplit(outdir)
if not split.scheme:
outdir = file_uri(str(outdir))
if "location" in f:
if f["location"].startswith("file://"):
path = uri_file_path(f["location"])
revmap_f = builder.pathmapper.reversemap(path)
if revmap_f:
f["location"] = revmap_f[1]
elif path == builder.outdir:
f["location"] = outdir
elif path.startswith(builder.outdir):
f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:])
return f
if "path" in f:
path = f["path"]
del f["path"]
revmap_f = builder.pathmapper.reversemap(path)
if revmap_f:
f["location"] = revmap_f[1]
return f
elif path.startswith(builder.outdir):
f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:])
return f
else:
raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input "
u"file pass through." % (path, builder.outdir))
raise WorkflowException(u"Output File object is missing both `location` and `path` fields: %s" % f)
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument("workflow_file", type=str,
help="CWL workflow specification file")
parser.add_argument("job_file", nargs='+',
help="One or more whitespace separated job files")
# directory to write any outputs to
# defaults to the current working directory
parser.add_argument("--outdir", type=str, default=os.getcwd(),
help="Directory to write the outputs to. "\
"Defaults to the current working directory.")
# if executed from command line, update args to those
if args is None:
args = sys.argv[1:]
options = parser.parse_args(args)
print("Options: " + str(options))
# create a cwltool object from the cwl workflow file
try:
tool = cwltool.load_tool.load_tool(
options.workflow_file,
makeDataCommonsTool,
kwargs={},
resolver=cwltool.resolver.tool_resolver
)
print("Tool:")
print(vars(tool))
except cwltool.process.UnsupportedRequirement as e:
print("UnsupportedRequirement")
# set up args for load_job_order
options.workflow = options.workflow_file
options.job_order = options.job_file
# set default basedir to be the cwd.
#Maybe set to None and let load_job_order determine it
options.basedir = os.getcwd()
options.tool_help = None
options.debug = True
# delayed import cwltool.main (circular dependency)
import cwltool.main
# load the job files
job, _ = cwltool.main.load_job_order(options, tool, sys.stdin)
print("Job: ")
pprint(job)
for inputname in job:
print("inputname: {}".format(inputname))
if inputname == "file":
filearg = job["file"]
print("filearg: {}".format(filearg))
if filearg.get("location"):
filearg["path"] = uri_file_path(filearg["location"])
kwargs = {
'basedir': options.basedir,
'outdir': options.basedir
}
jobiter = tool.job(job, None, **kwargs)
for jobrunner in jobiter:
if jobrunner:
jobrunner.run(**kwargs)
else:
print("")
def locToPath(p):
if "path" not in p and "location" in p:
p["path"] = uri_file_path(p["location"])
def loc_to_path(obj):
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
if obj["location"].startswith("file://"):
obj["path"] = uri_file_path(obj["location"])
def visit(
self,
obj: CWLObjectType,
stagedir: str,
basedir: str,
copy: bool = False,
staged: bool = False,
) -> None:
tgt = convert_pathsep_to_unix(
os.path.join(stagedir, cast(str, obj["basename"]))
)
if obj["location"] in self._pathmap:
return
if obj["class"] == "Directory":
location = cast(str, obj["location"])
if location.startswith("file://"):
resolved = uri_file_path(location)
else:
resolved = location
self._pathmap[location] = MapperEnt(
resolved, tgt, "WritableDirectory" if copy else "Directory", staged
)
if location.startswith("file://"):
staged = False
self.visitlisting(
cast(List[CWLObjectType], obj.get("listing", [])),
tgt,
basedir,
copy=copy,
staged=staged,
)
elif obj["class"] == "File":
path = cast(str, obj["location"])
ab = abspath(path, basedir)
if "contents" in obj and path.startswith("_:"):
self._pathmap[path] = MapperEnt(
obj["contents"],
tgt,
"CreateWritableFile" if copy else "CreateFile",
staged,
)
else:
with SourceLine(
obj,
"location",
ValidationException,
_logger.isEnabledFor(logging.DEBUG),
):
deref = ab
if urllib.parse.urlsplit(deref).scheme in ["http", "https"]:
deref = downloadHttpFile(path)
else:
# Dereference symbolic links
st = os.lstat(deref)
while stat.S_ISLNK(st.st_mode):
rl = os.readlink(deref)
deref = (
rl
if os.path.isabs(rl)
else os.path.join(os.path.dirname(deref), rl)
)
st = os.lstat(deref)
self._pathmap[path] = MapperEnt(
deref, tgt, "WritableFile" if copy else "File", staged
)
self.visitlisting(
cast(List[CWLObjectType], obj.get("secondaryFiles", [])),
stagedir,
basedir,
copy=copy,
staged=staged,
)
def revmap_file(builder, outdir, f):
# type: (Builder, str, Dict[str, Any]) -> Union[Dict[str, Any], None]
"""
Remap a file from internal path to external path.
For Docker, this maps from the path inside tho container to the path
outside the container. Recognizes files in the pathmapper or remaps
internal output directories to the external directory.
"""
split = urllib.parse.urlsplit(outdir)
if not split.scheme:
outdir = file_uri(str(outdir))
# builder.outdir is the inner (container/compute node) output directory
# outdir is the outer (host/storage system) output directory
if "location" in f and "path" not in f:
if f["location"].startswith("file://"):
f["path"] = convert_pathsep_to_unix(uri_file_path(f["location"]))
else:
return f
if "path" in f:
path = f["path"]
uripath = file_uri(path)
del f["path"]
if "basename" not in f:
f["basename"] = os.path.basename(path)
if not builder.pathmapper:
raise ValueError(
"Do not call revmap_file using a builder that doesn't have a pathmapper."
)
revmap_f = builder.pathmapper.reversemap(path)
if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith(
"Writable"
):
f["location"] = revmap_f[1]
elif (
uripath == outdir
or uripath.startswith(outdir + os.sep)
or uripath.startswith(outdir + "/")
):
f["location"] = file_uri(path)
elif (
path == builder.outdir
or path.startswith(builder.outdir + os.sep)
or path.startswith(builder.outdir + "/")
):
f["location"] = builder.fs_access.join(
outdir, path[len(builder.outdir) + 1 :]
)
elif not os.path.isabs(path):
f["location"] = builder.fs_access.join(outdir, path)
else:
raise WorkflowException(
"Output file path %s must be within designated output directory (%s) or an input "
"file pass through." % (path, builder.outdir)
)
return f
raise WorkflowException(
"Output File object is missing both 'location' " "and 'path' fields: %s" % f
)
def locToPath(p):
for field in ("path", "nameext", "nameroot", "dirname"):
if field in p:
del p[field]
if p["location"].startswith("file://"):
p["path"] = uri_file_path(p["location"])
def _realpath(ob): # type: (Dict[Text, Any]) -> None
if ob["location"].startswith("file:"):
ob["location"] = file_uri(
os.path.realpath(uri_file_path(ob["location"])))
if ob["location"].startswith("/"):
ob["location"] = os.path.realpath(ob["location"])
def abspath(src, basedir): # type: (Text, Text) -> Text
if src.startswith(u"file://"):
ab = unicode(uri_file_path(str(src)))
else:
ab = src if os.path.isabs(src) else os.path.join(basedir, src)
return ab
def loc_to_path(obj): # type: (Dict[str, Any]) -> None
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
if obj["location"].startswith("file://"):
obj["path"] = uri_file_path(obj["location"])
def locToPath(p):
if p["location"].startswith("file://"):
p["path"] = uri_file_path(p["location"])
def loc_to_path(obj: CWLObjectType) -> None:
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
if cast(str, obj["location"]).startswith("file://"):
obj["path"] = uri_file_path(cast(str, obj["location"]))
def abspath(src, basedir): # type: (Text, Text) -> Text
if src.startswith(u"file://"):
ab = unicode(uri_file_path(str(src)))
else:
ab = src if os.path.isabs(src) else os.path.join(basedir, src)
return ab
def locToPath(p):
if p["location"].startswith("file://"):
p["path"] = uri_file_path(p["location"])
def loc_to_path(obj):
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
if obj["location"].startswith("file://"):
obj["path"] = uri_file_path(obj["location"])
def locToPath(p):
for field in ("path", "nameext", "nameroot", "dirname"):
if field in p:
del p[field]
if p["location"].startswith("file://"):
p["path"] = uri_file_path(p["location"])
