def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
if srcobj["class"] == "File":
if "#" in src:
src = src[:src.index("#")]
if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "File")
if src not in self._pathmap:
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("", ab, fnPattern=self.file_pattern)
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
self._pathmap[src] = MapperEnt(ab, st.fn, "File")
elif src.startswith("_:") and "contents" in srcobj:
pass
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
if "secondaryFiles" in srcobj:
for l in srcobj["secondaryFiles"]:
self.visit(l, uploadfiles)
elif srcobj["class"] == "Directory":
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
else:
for l in srcobj["listing"]:
self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
if srcobj["class"] == "File":
if "#" in src:
src = src[:src.index("#")]
if isinstance(src,
basestring) and ArvPathMapper.pdh_path.match(src):
self._pathmap[src] = MapperEnt(
src, self.collection_pattern % src[5:], "File")
if src not in self._pathmap:
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("",
ab,
fnPattern=self.file_pattern)
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
self._pathmap[src] = MapperEnt(ab, st.fn, "File")
elif src.startswith("_:") and "contents" in srcobj:
pass
else:
raise WorkflowException("Input file path '%s' is invalid" %
st)
if "secondaryFiles" in srcobj:
for l in srcobj["secondaryFiles"]:
self.visit(l, uploadfiles)
elif srcobj["class"] == "Directory":
if isinstance(src,
basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(
src, self.collection_pattern % src[5:], "Directory")
for l in srcobj["listing"]:
self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
if "#" in src:
src = src[:src.index("#")]
if isinstance(src,
basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(
src, self.collection_pattern % urllib.parse.unquote(src[5:]),
srcobj["class"], True)
if arvados_cwl.util.collectionUUID in srcobj:
self.pdh_to_uuid[src.split(
"/", 1)[0][5:]] = srcobj[arvados_cwl.util.collectionUUID]
debug = logger.isEnabledFor(logging.DEBUG)
if src not in self._pathmap:
if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("",
ab,
fnPattern="keep:%s/%s",
dirPattern="keep:%s/%s",
raiseOSError=True)
with SourceLine(srcobj, "location", WorkflowException, debug):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
self._pathmap[src] = MapperEnt(
st.fn, self.collection_pattern %
urllib.parse.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException(
"Input file path '%s' is invalid" % st)
elif src.startswith("_:"):
if srcobj["class"] == "File" and "contents" not in srcobj:
raise WorkflowException(
"File literal '%s' is missing `contents`" % src)
if srcobj["class"] == "Directory" and "listing" not in srcobj:
raise WorkflowException(
"Directory literal '%s' is missing `listing`" % src)
elif src.startswith("http:") or src.startswith("https:"):
keepref = http_to_keep(self.arvrunner.api,
self.arvrunner.project_uuid, src)
logger.info("%s is %s", src, keepref)
self._pathmap[src] = MapperEnt(keepref, keepref,
srcobj["class"], True)
else:
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug):
for l in srcobj.get("secondaryFiles", []):
self.visit(l, uploadfiles)
with SourceLine(srcobj, "listing", WorkflowException, debug):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
def listdir(self, fn): # type: (unicode) -> List[unicode]
collection, rest = self.get_collection(fn)
if rest:
dir = collection.find(rest)
else:
dir = collection
if collection:
return [abspath(l, fn) for l in dir.keys()]
else:
return super(CollectionFsAccess, self).listdir(fn)
def listdir(self, fn): # type: (unicode) -> List[unicode]
collection, rest = self.get_collection(fn)
if collection:
if rest:
dir = collection.find(rest)
else:
dir = collection
return [abspath(l, fn) for l in dir.keys()]
else:
return super(CollectionFsAccess, self).listdir(fn)
def listdir(self, fn): # type: (unicode) -> List[unicode]
collection, rest = self.get_collection(fn)
if collection:
if rest:
dir = collection.find(rest)
else:
dir = collection
if dir is None:
raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash()))
if not isinstance(dir, arvados.collection.RichCollectionBase):
raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash()))
return [abspath(l, fn) for l in dir.keys()]
else:
return super(CollectionFsAccess, self).listdir(fn)
def listdir(self, fn): # type: (unicode) -> List[unicode]
collection, rest = self.get_collection(fn)
if collection is not None:
if rest:
dir = collection.find(rest)
else:
dir = collection
if dir is None:
raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash()))
if not isinstance(dir, arvados.collection.RichCollectionBase):
raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash()))
return [abspath(l, fn) for l in list(dir.keys())]
else:
return super(CollectionFsAccess, self).listdir(fn)
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
if "#" in src:
src = src[:src.index("#")]
if isinstance(src,
basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(
src, self.collection_pattern % urllib.unquote(src[5:]),
srcobj["class"], True)
if src not in self._pathmap:
if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("",
ab,
fnPattern="keep:%s/%s",
dirPattern="keep:%s/%s",
raiseOSError=True)
with SourceLine(srcobj, "location", WorkflowException):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
self._pathmap[src] = MapperEnt(
st.fn, self.collection_pattern %
urllib.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException(
"Input file path '%s' is invalid" % st)
elif src.startswith("_:"):
if srcobj["class"] == "File" and "contents" not in srcobj:
raise WorkflowException(
"File literal '%s' is missing `contents`" % src)
if srcobj["class"] == "Directory" and "listing" not in srcobj:
raise WorkflowException(
"Directory literal '%s' is missing `listing`" % src)
else:
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
with SourceLine(srcobj, "secondaryFiles", WorkflowException):
for l in srcobj.get("secondaryFiles", []):
self.visit(l, uploadfiles)
with SourceLine(srcobj, "listing", WorkflowException):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
if "#" in src:
src = src[:src.index("#")]
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True)
if arvados_cwl.util.collectionUUID in srcobj:
self.pdh_to_uuid[src.split("/", 1)[0][5:]] = srcobj[arvados_cwl.util.collectionUUID]
debug = logger.isEnabledFor(logging.DEBUG)
if src not in self._pathmap:
if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("", ab,
fnPattern="keep:%s/%s",
dirPattern="keep:%s/%s",
raiseOSError=True)
with SourceLine(srcobj, "location", WorkflowException, debug):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
elif src.startswith("_:"):
if srcobj["class"] == "File" and "contents" not in srcobj:
raise WorkflowException("File literal '%s' is missing `contents`" % src)
if srcobj["class"] == "Directory" and "listing" not in srcobj:
raise WorkflowException("Directory literal '%s' is missing `listing`" % src)
elif src.startswith("http:") or src.startswith("https:"):
keepref = http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src)
logger.info("%s is %s", src, keepref)
self._pathmap[src] = MapperEnt(keepref, keepref, srcobj["class"], True)
else:
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug):
for l in srcobj.get("secondaryFiles", []):
self.visit(l, uploadfiles)
with SourceLine(srcobj, "listing", WorkflowException, debug):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
if "#" in src:
src = src[:src.index("#")]
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
if src not in self._pathmap:
if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("", ab,
fnPattern="keep:%s/%s",
dirPattern="keep:%s/%s")
with SourceLine(srcobj, "location", WorkflowException):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
elif src.startswith("_:"):
if srcobj["class"] == "File" and "contents" not in srcobj:
raise WorkflowException("File literal '%s' is missing `contents`" % src)
if srcobj["class"] == "Directory" and "listing" not in srcobj:
raise WorkflowException("Directory literal '%s' is missing `listing`" % src)
else:
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
with SourceLine(srcobj, "secondaryFiles", WorkflowException):
for l in srcobj.get("secondaryFiles", []):
self.visit(l, uploadfiles)
with SourceLine(srcobj, "listing", WorkflowException):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
