def toilStageFiles(fileStore, cwljob, outdir, index, existing, export):
"""Copy input files out of the global file store and update location and
path."""
jobfiles = [] # type: List[Dict[Text, Any]]
collectFilesAndDirs(cwljob, jobfiles)
pm = ToilPathMapper(jobfiles, "", outdir, separateDirs=False, stage_listing=True)
for f, p in pm.items():
if not p.staged:
continue
if not os.path.exists(os.path.dirname(p.target)):
os.makedirs(os.path.dirname(p.target), 0o0755)
if p.type == "File":
fileStore.exportFile(p.resolved[7:], "file://" + p.target)
elif p.type == "Directory" and not os.path.exists(p.target):
os.makedirs(p.target, 0o0755)
elif p.type == "CreateFile":
with open(p.target, "wb") as n:
n.write(p.resolved.encode("utf-8"))
def _check_adjust(f):
f["location"] = schema_salad.ref_resolver.file_uri(pm.mapper(f["location"])[1])
if "contents" in f:
del f["contents"]
return f
visit_class(cwljob, ("File", "Directory"), _check_adjust)
def toilStageFiles(fileStore, cwljob, outdir, index, existing, export):
"""Copy input files out of the global file store and update location and
path."""
jobfiles = [] # type: List[Dict[Text, Any]]
collectFilesAndDirs(cwljob, jobfiles)
pm = ToilPathMapper(jobfiles, "", outdir, separateDirs=False, stage_listing=True)
for f, p in pm.items():
if not p.staged:
continue
if not os.path.exists(os.path.dirname(p.target)):
os.makedirs(os.path.dirname(p.target), 0o0755)
if p.type == "File":
fileStore.exportFile(p.resolved[7:], "file://" + p.target)
elif p.type == "Directory" and not os.path.exists(p.target):
os.makedirs(p.target, 0o0755)
elif p.type == "CreateFile":
with open(p.target, "wb") as n:
n.write(p.resolved.encode("utf-8"))
def _check_adjust(f):
f["location"] = schema_salad.ref_resolver.file_uri(pm.mapper(f["location"])[1])
if "contents" in f:
del f["contents"]
return f
visit_class(cwljob, ("File", "Directory"), _check_adjust)
def toilStageFiles(fileStore,
cwljob,
outdir,
index,
existing,
export,
destBucket=None):
"""Copy input files out of the global file store and update location and
path."""
jobfiles = [] # type: List[Dict[Text, Any]]
collectFilesAndDirs(cwljob, jobfiles)
pm = ToilPathMapper(jobfiles,
"",
outdir,
separateDirs=False,
stage_listing=True)
for f, p in pm.items():
if not p.staged:
continue
# Deal with bucket exports
if destBucket:
# Directories don't need to be created if we're exporting to
# a bucket
if p.type == "File":
# Remove the staging directory from the filepath and
# form the destination URL
unstageTargetPath = p.target[len(outdir):]
destUrl = '/'.join(
s.strip('/') for s in [destBucket, unstageTargetPath])
fileStore.exportFile(p.resolved[7:], destUrl)
continue
if not os.path.exists(os.path.dirname(p.target)):
os.makedirs(os.path.dirname(p.target), 0o0755)
if p.type == "File":
fileStore.exportFile(p.resolved[7:], "file://" + p.target)
elif p.type == "Directory" and not os.path.exists(p.target):
os.makedirs(p.target, 0o0755)
elif p.type == "CreateFile":
with open(p.target, "wb") as n:
n.write(p.resolved.encode("utf-8"))
def _check_adjust(f):
f["location"] = schema_salad.ref_resolver.file_uri(
pm.mapper(f["location"])[1])
if "contents" in f:
del f["contents"]
return f
visit_class(cwljob, ("File", "Directory"), _check_adjust)