def __contains__(self, k): if k in self._entries: return True if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)): return False try: e = self.inodes.add_entry(CollectionDirectory( self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: self._entries[k] = e else: self.inodes.del_entry(e) return True else: self.inodes.invalidate_entry(self.inode, k) self.inodes.del_entry(e) return False except Exception as ex: _logger.debug('arv-mount exception keep %s', ex) self.inodes.del_entry(e) return False
def update(self): try: if self.collection_object is not None and portable_data_hash_pattern.match(self.collection_locator): return True with llfuse.lock_released: new_collection_object = self.api.collections().get( uuid=self.collection_locator ).execute(num_retries=self.num_retries) if "portable_data_hash" not in new_collection_object: new_collection_object["portable_data_hash"] = new_collection_object["uuid"] # end with llfuse.lock_released, re-acquire lock if self.collection_object is None or self.collection_object["portable_data_hash"] != new_collection_object["portable_data_hash"]: self.new_collection(new_collection_object) self.fresh() return True except apiclient.errors.HttpError as e: if e.resp.status == 404: _logger.warn("arv-mount %s: not found", self.collection_locator) else: _logger.error("arv-mount %s: error", self.collection_locator) _logger.exception(detail) except arvados.errors.ArgumentError as detail: _logger.warning("arv-mount %s: error %s", self.collection_locator, detail) if self.collection_object is not None and "manifest_text" in self.collection_object: _logger.warning("arv-mount manifest_text is: %s", self.collection_object["manifest_text"]) except Exception as detail: _logger.error("arv-mount %s: error", self.collection_locator) if self.collection_object is not None and "manifest_text" in self.collection_object: _logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"]) _logger.exception(detail) return False
def __contains__(self, k): if k in self._entries: return True if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)): return False try: e = None e = self.inodes.add_entry(CollectionDirectory( self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: self._entries[k] = e else: self.inodes.del_entry(e) return True else: self.inodes.invalidate_entry(self, k) self.inodes.del_entry(e) return False except Exception as ex: _logger.exception("arv-mount lookup '%s':", k) if e is not None: self.inodes.del_entry(e) return False
def __contains__(self, k): if k in self._entries: return True if not portable_data_hash_pattern.match(k) and not uuid_pattern.match( k): return False try: e = self.inodes.add_entry( CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: self._entries[k] = e else: self.inodes.del_entry(e) return True else: self.inodes.del_entry(e) return False except Exception as e: _logger.debug('arv-mount exception keep %s', e) self.inodes.del_entry(e) return False
def update(self): try: if self.collection_object is not None and portable_data_hash_pattern.match( self.collection_locator): return True if self.collection_locator is None: self.fresh() return True with llfuse.lock_released: coll_reader = arvados.CollectionReader( self.collection_locator, self.api, self.api.keep, num_retries=self.num_retries) new_collection_object = coll_reader.api_response() or {} # If the Collection only exists in Keep, there will be no API # response. Fill in the fields we need. if 'uuid' not in new_collection_object: new_collection_object['uuid'] = self.collection_locator if "portable_data_hash" not in new_collection_object: new_collection_object[ "portable_data_hash"] = new_collection_object["uuid"] if 'manifest_text' not in new_collection_object: new_collection_object[ 'manifest_text'] = coll_reader.manifest_text() coll_reader.normalize() # end with llfuse.lock_released, re-acquire lock if self.collection_object is None or self.collection_object[ "portable_data_hash"] != new_collection_object[ "portable_data_hash"]: self.new_collection(new_collection_object, coll_reader) self.fresh() return True except arvados.errors.NotFoundError: _logger.exception("arv-mount %s: error", self.collection_locator) except arvados.errors.ArgumentError as detail: _logger.warning("arv-mount %s: error %s", self.collection_locator, detail) if self.collection_object is not None and "manifest_text" in self.collection_object: _logger.warning("arv-mount manifest_text is: %s", self.collection_object["manifest_text"]) except Exception: _logger.exception("arv-mount %s: error", self.collection_locator) if self.collection_object is not None and "manifest_text" in self.collection_object: _logger.error("arv-mount manifest_text is: %s", self.collection_object["manifest_text"]) return False
def createDirectory(self, i): if collection_uuid_pattern.match(i['uuid']): return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i) elif group_uuid_pattern.match(i['uuid']): return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time) elif link_uuid_pattern.match(i['uuid']): if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']): return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid']) else: return None elif uuid_pattern.match(i['uuid']): return ObjectFile(self.parent_inode, i) else: return None
def createDirectory(self, i): if collection_uuid_pattern.match(i['uuid']): return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i) elif group_uuid_pattern.match(i['uuid']): return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time) elif link_uuid_pattern.match(i['uuid']): if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']): return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid']) else: return None elif uuid_pattern.match(i['uuid']): return ObjectFile(self.parent_inode, i) else: return None
def __contains__(self, k): if k in self._entries: return True if not portable_data_hash_pattern.match(k) and ( self.pdh_only or not uuid_pattern.match(k)): return False try: e = None if group_uuid_pattern.match(k): project = self.api.groups().list( filters=[['group_class', '=', 'project'], ["uuid", "=", k] ]).execute(num_retries=self.num_retries) if project[u'items_available'] == 0: return False e = self.inodes.add_entry( ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0])) else: import sys e = self.inodes.add_entry( CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: self._entries[k] = e else: self.inodes.del_entry(e) return True else: self.inodes.invalidate_entry(self, k) self.inodes.del_entry(e) return False except Exception as ex: _logger.exception("arv-mount lookup '%s':", k) if e is not None: self.inodes.del_entry(e) return False
def __contains__(self, k): if k in self._entries: return True if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)): return False try: e = None if group_uuid_pattern.match(k): project = self.api.groups().list( filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries) if project[u'items_available'] == 0: return False e = self.inodes.add_entry(ProjectDirectory( self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0])) else: e = self.inodes.add_entry(CollectionDirectory( self.inode, self.inodes, self.api, self.num_retries, k)) if e.update(): if k not in self._entries: self._entries[k] = e else: self.inodes.del_entry(e) return True else: self.inodes.invalidate_entry(self, k) self.inodes.del_entry(e) return False except Exception as ex: _logger.exception("arv-mount lookup '%s':", k) if e is not None: self.inodes.del_entry(e) return False
def update(self, to_record_version=None): try: if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator): return True if self.collection_locator is None: self.fresh() return True try: with llfuse.lock_released: self._updating_lock.acquire() if not self.stale(): return _logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version) if self.collection is not None: if self.collection.known_past_version(to_record_version): _logger.debug("%s already processed %s", self.collection_locator, to_record_version) else: self.collection.update() else: if uuid_pattern.match(self.collection_locator): coll_reader = arvados.collection.Collection( self.collection_locator, self.api, self.api.keep, num_retries=self.num_retries) else: coll_reader = arvados.collection.CollectionReader( self.collection_locator, self.api, self.api.keep, num_retries=self.num_retries) new_collection_record = coll_reader.api_response() or {} # If the Collection only exists in Keep, there will be no API # response. Fill in the fields we need. if 'uuid' not in new_collection_record: new_collection_record['uuid'] = self.collection_locator if "portable_data_hash" not in new_collection_record: new_collection_record["portable_data_hash"] = new_collection_record["uuid"] if 'manifest_text' not in new_collection_record: new_collection_record['manifest_text'] = coll_reader.manifest_text() if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"): self.new_collection(new_collection_record, coll_reader) self._manifest_size = len(coll_reader.manifest_text()) _logger.debug("%s manifest_size %i", self, self._manifest_size) # end with llfuse.lock_released, re-acquire lock self.fresh() return True finally: self._updating_lock.release() except arvados.errors.NotFoundError as e: _logger.error("Error fetching collection '%s': %s", self.collection_locator, e) except arvados.errors.ArgumentError as detail: _logger.warning("arv-mount %s: error %s", self.collection_locator, detail) if self.collection_record is not None and "manifest_text" in self.collection_record: _logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"]) except Exception: _logger.exception("arv-mount %s: error", self.collection_locator) if self.collection_record is not None and "manifest_text" in self.collection_record: _logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"]) self.invalidate() return False
def update(self, to_record_version=None): try: if self.collection_record is not None and portable_data_hash_pattern.match( self.collection_locator): return True if self.collection_locator is None: self.fresh() return True try: with llfuse.lock_released: self._updating_lock.acquire() if not self.stale(): return _logger.debug( "Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version) if self.collection is not None: if self.collection.known_past_version( to_record_version): _logger.debug("%s already processed %s", self.collection_locator, to_record_version) else: self.collection.update() else: if uuid_pattern.match(self.collection_locator): coll_reader = arvados.collection.Collection( self.collection_locator, self.api, self.api.keep, num_retries=self.num_retries) else: coll_reader = arvados.collection.CollectionReader( self.collection_locator, self.api, self.api.keep, num_retries=self.num_retries) new_collection_record = coll_reader.api_response( ) or {} # If the Collection only exists in Keep, there will be no API # response. Fill in the fields we need. if 'uuid' not in new_collection_record: new_collection_record[ 'uuid'] = self.collection_locator if "portable_data_hash" not in new_collection_record: new_collection_record[ "portable_data_hash"] = new_collection_record[ "uuid"] if 'manifest_text' not in new_collection_record: new_collection_record[ 'manifest_text'] = coll_reader.manifest_text() if self.collection_record is None or self.collection_record[ "portable_data_hash"] != new_collection_record.get( "portable_data_hash"): self.new_collection(new_collection_record, coll_reader) self._manifest_size = len(coll_reader.manifest_text()) _logger.debug("%s manifest_size %i", self, self._manifest_size) # end with llfuse.lock_released, re-acquire lock self.fresh() return True finally: self._updating_lock.release() except arvados.errors.NotFoundError as e: _logger.error("Error fetching collection '%s': %s", self.collection_locator, e) except arvados.errors.ArgumentError as detail: _logger.warning("arv-mount %s: error %s", self.collection_locator, detail) if self.collection_record is not None and "manifest_text" in self.collection_record: _logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"]) except Exception: _logger.exception("arv-mount %s: error", self.collection_locator) if self.collection_record is not None and "manifest_text" in self.collection_record: _logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"]) self.invalidate() return False