def visit(self, srcobj, uploadfiles): src = srcobj["location"] if srcobj["class"] == "File": if "#" in src: src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src): self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "File") if src not in self._pathmap: # Local FS ref, may need to be uploaded or may be on keep # mount. ab = abspath(src, self.input_basedir) st = arvados.commands.run.statfile("", ab, fnPattern=self.file_pattern) if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): self._pathmap[src] = MapperEnt(ab, st.fn, "File") elif src.startswith("_:") and "contents" in srcobj: pass else: raise WorkflowException("Input file path '%s' is invalid" % st) if "secondaryFiles" in srcobj: for l in srcobj["secondaryFiles"]: self.visit(l, uploadfiles) elif srcobj["class"] == "Directory": if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory") else: for l in srcobj["listing"]: self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles): src = srcobj["location"] if srcobj["class"] == "File": if "#" in src: src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src): self._pathmap[src] = MapperEnt( src, self.collection_pattern % src[5:], "File") if src not in self._pathmap: # Local FS ref, may need to be uploaded or may be on keep # mount. ab = abspath(src, self.input_basedir) st = arvados.commands.run.statfile("", ab, fnPattern=self.file_pattern) if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): self._pathmap[src] = MapperEnt(ab, st.fn, "File") elif src.startswith("_:") and "contents" in srcobj: pass else: raise WorkflowException("Input file path '%s' is invalid" % st) if "secondaryFiles" in srcobj: for l in srcobj["secondaryFiles"]: self.visit(l, uploadfiles) elif srcobj["class"] == "Directory": if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): self._pathmap[src] = MapperEnt( src, self.collection_pattern % src[5:], "Directory") for l in srcobj["listing"]: self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles): src = srcobj["location"] if "#" in src: src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): self._pathmap[src] = MapperEnt( src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True) if arvados_cwl.util.collectionUUID in srcobj: self.pdh_to_uuid[src.split( "/", 1)[0][5:]] = srcobj[arvados_cwl.util.collectionUUID] debug = logger.isEnabledFor(logging.DEBUG) if src not in self._pathmap: if src.startswith("file:"): # Local FS ref, may need to be uploaded or may be on keep # mount. ab = abspath(src, self.input_basedir) st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s", dirPattern="keep:%s/%s", raiseOSError=True) with SourceLine(srcobj, "location", WorkflowException, debug): if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): self._pathmap[src] = MapperEnt( st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True) else: raise WorkflowException( "Input file path '%s' is invalid" % st) elif src.startswith("_:"): if srcobj["class"] == "File" and "contents" not in srcobj: raise WorkflowException( "File literal '%s' is missing `contents`" % src) if srcobj["class"] == "Directory" and "listing" not in srcobj: raise WorkflowException( "Directory literal '%s' is missing `listing`" % src) elif src.startswith("http:") or src.startswith("https:"): keepref = http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src) logger.info("%s is %s", src, keepref) self._pathmap[src] = MapperEnt(keepref, keepref, srcobj["class"], True) else: self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True) with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug): for l in srcobj.get("secondaryFiles", []): self.visit(l, uploadfiles) with SourceLine(srcobj, "listing", WorkflowException, debug): for l in srcobj.get("listing", []): self.visit(l, uploadfiles)
def listdir(self, fn): # type: (unicode) -> List[unicode] collection, rest = self.get_collection(fn) if rest: dir = collection.find(rest) else: dir = collection if collection: return [abspath(l, fn) for l in dir.keys()] else: return super(CollectionFsAccess, self).listdir(fn)
def listdir(self, fn): # type: (unicode) -> List[unicode] collection, rest = self.get_collection(fn) if collection: if rest: dir = collection.find(rest) else: dir = collection return [abspath(l, fn) for l in dir.keys()] else: return super(CollectionFsAccess, self).listdir(fn)
def listdir(self, fn): # type: (unicode) -> List[unicode] collection, rest = self.get_collection(fn) if collection: if rest: dir = collection.find(rest) else: dir = collection if dir is None: raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash())) if not isinstance(dir, arvados.collection.RichCollectionBase): raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash())) return [abspath(l, fn) for l in dir.keys()] else: return super(CollectionFsAccess, self).listdir(fn)
def listdir(self, fn): # type: (unicode) -> List[unicode] collection, rest = self.get_collection(fn) if collection is not None: if rest: dir = collection.find(rest) else: dir = collection if dir is None: raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash())) if not isinstance(dir, arvados.collection.RichCollectionBase): raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash())) return [abspath(l, fn) for l in list(dir.keys())] else: return super(CollectionFsAccess, self).listdir(fn)
def visit(self, srcobj, uploadfiles): src = srcobj["location"] if "#" in src: src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): self._pathmap[src] = MapperEnt( src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True) if src not in self._pathmap: if src.startswith("file:"): # Local FS ref, may need to be uploaded or may be on keep # mount. ab = abspath(src, self.input_basedir) st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s", dirPattern="keep:%s/%s", raiseOSError=True) with SourceLine(srcobj, "location", WorkflowException): if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): self._pathmap[src] = MapperEnt( st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True) else: raise WorkflowException( "Input file path '%s' is invalid" % st) elif src.startswith("_:"): if srcobj["class"] == "File" and "contents" not in srcobj: raise WorkflowException( "File literal '%s' is missing `contents`" % src) if srcobj["class"] == "Directory" and "listing" not in srcobj: raise WorkflowException( "Directory literal '%s' is missing `listing`" % src) else: self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True) with SourceLine(srcobj, "secondaryFiles", WorkflowException): for l in srcobj.get("secondaryFiles", []): self.visit(l, uploadfiles) with SourceLine(srcobj, "listing", WorkflowException): for l in srcobj.get("listing", []): self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles): src = srcobj["location"] if "#" in src: src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True) if arvados_cwl.util.collectionUUID in srcobj: self.pdh_to_uuid[src.split("/", 1)[0][5:]] = srcobj[arvados_cwl.util.collectionUUID] debug = logger.isEnabledFor(logging.DEBUG) if src not in self._pathmap: if src.startswith("file:"): # Local FS ref, may need to be uploaded or may be on keep # mount. ab = abspath(src, self.input_basedir) st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s", dirPattern="keep:%s/%s", raiseOSError=True) with SourceLine(srcobj, "location", WorkflowException, debug): if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True) else: raise WorkflowException("Input file path '%s' is invalid" % st) elif src.startswith("_:"): if srcobj["class"] == "File" and "contents" not in srcobj: raise WorkflowException("File literal '%s' is missing `contents`" % src) if srcobj["class"] == "Directory" and "listing" not in srcobj: raise WorkflowException("Directory literal '%s' is missing `listing`" % src) elif src.startswith("http:") or src.startswith("https:"): keepref = http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src) logger.info("%s is %s", src, keepref) self._pathmap[src] = MapperEnt(keepref, keepref, srcobj["class"], True) else: self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True) with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug): for l in srcobj.get("secondaryFiles", []): self.visit(l, uploadfiles) with SourceLine(srcobj, "listing", WorkflowException, debug): for l in srcobj.get("listing", []): self.visit(l, uploadfiles)
def visit(self, srcobj, uploadfiles): src = srcobj["location"] if "#" in src: src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True) if src not in self._pathmap: if src.startswith("file:"): # Local FS ref, may need to be uploaded or may be on keep # mount. ab = abspath(src, self.input_basedir) st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s", dirPattern="keep:%s/%s") with SourceLine(srcobj, "location", WorkflowException): if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True) else: raise WorkflowException("Input file path '%s' is invalid" % st) elif src.startswith("_:"): if srcobj["class"] == "File" and "contents" not in srcobj: raise WorkflowException("File literal '%s' is missing `contents`" % src) if srcobj["class"] == "Directory" and "listing" not in srcobj: raise WorkflowException("Directory literal '%s' is missing `listing`" % src) else: self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True) with SourceLine(srcobj, "secondaryFiles", WorkflowException): for l in srcobj.get("secondaryFiles", []): self.visit(l, uploadfiles) with SourceLine(srcobj, "listing", WorkflowException): for l in srcobj.get("listing", []): self.visit(l, uploadfiles)