Beispiel #1
0
    def __init__(self,
                 parent_inode,
                 inodes,
                 api,
                 num_retries,
                 collection_record=None,
                 explicit_collection=None):
        super(CollectionDirectory, self).__init__(parent_inode, inodes,
                                                  api.config, None)
        self.api = api
        self.num_retries = num_retries
        self.collection_record_file = None
        self.collection_record = None
        self._poll = True
        try:
            self._poll_time = (
                api._rootDesc.get('blobSignatureTtl', 60 * 60 * 2) // 2)
        except:
            _logger.debug(
                "Error getting blobSignatureTtl from discovery document: %s",
                sys.exc_info()[0])
            self._poll_time = 60 * 60

        if isinstance(collection_record, dict):
            self.collection_locator = collection_record['uuid']
            self._mtime = convertTime(collection_record.get('modified_at'))
        else:
            self.collection_locator = collection_record
            self._mtime = 0
        self._manifest_size = 0
        if self.collection_locator:
            self._writable = (uuid_pattern.match(self.collection_locator)
                              is not None)
        self._updating_lock = threading.Lock()
Beispiel #2
0
    def __contains__(self, k):
        if k in self._entries:
            return True

        if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
            return False

        try:
            e = self.inodes.add_entry(CollectionDirectory(
                    self.inode, self.inodes, self.api, self.num_retries, k))

            if e.update():
                if k not in self._entries:
                    self._entries[k] = e
                else:
                    self.inodes.del_entry(e)
                return True
            else:
                self.inodes.invalidate_entry(self.inode, k)
                self.inodes.del_entry(e)
                return False
        except Exception as ex:
            _logger.debug('arv-mount exception keep %s', ex)
            self.inodes.del_entry(e)
            return False
Beispiel #3
0
    def __contains__(self, k):
        if k in self._entries:
            return True

        if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
            return False

        try:
            e = None
            e = self.inodes.add_entry(CollectionDirectory(
                    self.inode, self.inodes, self.api, self.num_retries, k))

            if e.update():
                if k not in self._entries:
                    self._entries[k] = e
                else:
                    self.inodes.del_entry(e)
                return True
            else:
                self.inodes.invalidate_entry(self, k)
                self.inodes.del_entry(e)
                return False
        except Exception as ex:
            _logger.exception("arv-mount lookup '%s':", k)
            if e is not None:
                self.inodes.del_entry(e)
            return False
Beispiel #4
0
    def __contains__(self, k):
        if k in self._entries:
            return True

        if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(
                k):
            return False

        try:
            e = self.inodes.add_entry(
                CollectionDirectory(self.inode, self.inodes, self.api,
                                    self.num_retries, k))

            if e.update():
                if k not in self._entries:
                    self._entries[k] = e
                else:
                    self.inodes.del_entry(e)
                return True
            else:
                self.inodes.del_entry(e)
                return False
        except Exception as e:
            _logger.debug('arv-mount exception keep %s', e)
            self.inodes.del_entry(e)
            return False
Beispiel #5
0
 def createDirectory(self, i):
     if collection_uuid_pattern.match(i['uuid']):
         return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
     elif group_uuid_pattern.match(i['uuid']):
         return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
     elif link_uuid_pattern.match(i['uuid']):
         if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
             return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
         else:
             return None
     elif uuid_pattern.match(i['uuid']):
         return ObjectFile(self.parent_inode, i)
     else:
         return None
Beispiel #6
0
 def createDirectory(self, i):
     if collection_uuid_pattern.match(i['uuid']):
         return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i)
     elif group_uuid_pattern.match(i['uuid']):
         return ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i, self._poll, self._poll_time)
     elif link_uuid_pattern.match(i['uuid']):
         if i['head_kind'] == 'arvados#collection' or portable_data_hash_pattern.match(i['head_uuid']):
             return CollectionDirectory(self.inode, self.inodes, self.api, self.num_retries, i['head_uuid'])
         else:
             return None
     elif uuid_pattern.match(i['uuid']):
         return ObjectFile(self.parent_inode, i)
     else:
         return None
Beispiel #7
0
 def __init__(self, parent_inode, inodes, api, num_retries, collection_record=None, explicit_collection=None):
     super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
     self.api = api
     self.num_retries = num_retries
     self.collection_record_file = None
     self.collection_record = None
     if isinstance(collection_record, dict):
         self.collection_locator = collection_record['uuid']
         self._mtime = convertTime(collection_record.get('modified_at'))
     else:
         self.collection_locator = collection_record
         self._mtime = 0
     self._manifest_size = 0
     if self.collection_locator:
         self._writable = (uuid_pattern.match(self.collection_locator) is not None)
     self._updating_lock = threading.Lock()
Beispiel #8
0
    def __contains__(self, k):
        if k in self._entries:
            return True

        if not portable_data_hash_pattern.match(k) and (
                self.pdh_only or not uuid_pattern.match(k)):
            return False

        try:
            e = None

            if group_uuid_pattern.match(k):
                project = self.api.groups().list(
                    filters=[['group_class', '=', 'project'], ["uuid", "=", k]
                             ]).execute(num_retries=self.num_retries)
                if project[u'items_available'] == 0:
                    return False
                e = self.inodes.add_entry(
                    ProjectDirectory(self.inode, self.inodes, self.api,
                                     self.num_retries, project[u'items'][0]))
            else:
                import sys
                e = self.inodes.add_entry(
                    CollectionDirectory(self.inode, self.inodes, self.api,
                                        self.num_retries, k))

            if e.update():
                if k not in self._entries:
                    self._entries[k] = e
                else:
                    self.inodes.del_entry(e)
                return True
            else:
                self.inodes.invalidate_entry(self, k)
                self.inodes.del_entry(e)
                return False
        except Exception as ex:
            _logger.exception("arv-mount lookup '%s':", k)
            if e is not None:
                self.inodes.del_entry(e)
            return False
Beispiel #9
0
    def __init__(self, parent_inode, inodes, api, num_retries, collection_record=None, explicit_collection=None):
        super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
        self.api = api
        self.num_retries = num_retries
        self.collection_record_file = None
        self.collection_record = None
        self._poll = True
        try:
            self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
        except:
            _logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0])
            self._poll_time = 60*60

        if isinstance(collection_record, dict):
            self.collection_locator = collection_record['uuid']
            self._mtime = convertTime(collection_record.get('modified_at'))
        else:
            self.collection_locator = collection_record
            self._mtime = 0
        self._manifest_size = 0
        if self.collection_locator:
            self._writable = (uuid_pattern.match(self.collection_locator) is not None)
        self._updating_lock = threading.Lock()
Beispiel #10
0
 def __init__(self,
              parent_inode,
              inodes,
              api,
              num_retries,
              collection_record=None,
              explicit_collection=None):
     super(CollectionDirectory, self).__init__(parent_inode, inodes, None)
     self.api = api
     self.num_retries = num_retries
     self.collection_record_file = None
     self.collection_record = None
     if isinstance(collection_record, dict):
         self.collection_locator = collection_record['uuid']
         self._mtime = convertTime(collection_record.get('modified_at'))
     else:
         self.collection_locator = collection_record
         self._mtime = 0
     self._manifest_size = 0
     if self.collection_locator:
         self._writable = (uuid_pattern.match(self.collection_locator)
                           is not None)
     self._updating_lock = threading.Lock()
Beispiel #11
0
    def __contains__(self, k):
        if k in self._entries:
            return True

        if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
            return False

        try:
            e = None

            if group_uuid_pattern.match(k):
                project = self.api.groups().list(
                    filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
                if project[u'items_available'] == 0:
                    return False
                e = self.inodes.add_entry(ProjectDirectory(
                    self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
            else:
                e = self.inodes.add_entry(CollectionDirectory(
                        self.inode, self.inodes, self.api, self.num_retries, k))

            if e.update():
                if k not in self._entries:
                    self._entries[k] = e
                else:
                    self.inodes.del_entry(e)
                return True
            else:
                self.inodes.invalidate_entry(self, k)
                self.inodes.del_entry(e)
                return False
        except Exception as ex:
            _logger.exception("arv-mount lookup '%s':", k)
            if e is not None:
                self.inodes.del_entry(e)
            return False
Beispiel #12
0
    def update(self, to_record_version=None):
        try:
            if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator):
                return True

            if self.collection_locator is None:
                self.fresh()
                return True

            try:
                with llfuse.lock_released:
                    self._updating_lock.acquire()
                    if not self.stale():
                        return

                    _logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version)
                    if self.collection is not None:
                        if self.collection.known_past_version(to_record_version):
                            _logger.debug("%s already processed %s", self.collection_locator, to_record_version)
                        else:
                            self.collection.update()
                    else:
                        if uuid_pattern.match(self.collection_locator):
                            coll_reader = arvados.collection.Collection(
                                self.collection_locator, self.api, self.api.keep,
                                num_retries=self.num_retries)
                        else:
                            coll_reader = arvados.collection.CollectionReader(
                                self.collection_locator, self.api, self.api.keep,
                                num_retries=self.num_retries)
                        new_collection_record = coll_reader.api_response() or {}
                        # If the Collection only exists in Keep, there will be no API
                        # response.  Fill in the fields we need.
                        if 'uuid' not in new_collection_record:
                            new_collection_record['uuid'] = self.collection_locator
                        if "portable_data_hash" not in new_collection_record:
                            new_collection_record["portable_data_hash"] = new_collection_record["uuid"]
                        if 'manifest_text' not in new_collection_record:
                            new_collection_record['manifest_text'] = coll_reader.manifest_text()

                        if self.collection_record is None or self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"):
                            self.new_collection(new_collection_record, coll_reader)

                        self._manifest_size = len(coll_reader.manifest_text())
                        _logger.debug("%s manifest_size %i", self, self._manifest_size)
                # end with llfuse.lock_released, re-acquire lock

                self.fresh()
                return True
            finally:
                self._updating_lock.release()
        except arvados.errors.NotFoundError as e:
            _logger.error("Error fetching collection '%s': %s", self.collection_locator, e)
        except arvados.errors.ArgumentError as detail:
            _logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
            if self.collection_record is not None and "manifest_text" in self.collection_record:
                _logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
        except Exception:
            _logger.exception("arv-mount %s: error", self.collection_locator)
            if self.collection_record is not None and "manifest_text" in self.collection_record:
                _logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
        self.invalidate()
        return False
Beispiel #13
0
 def want_event_subscribe(self):
     return (uuid_pattern.match(self.collection_locator) is not None)
Beispiel #14
0
    def update(self, to_record_version=None):
        try:
            if self.collection_record is not None and portable_data_hash_pattern.match(
                    self.collection_locator):
                return True

            if self.collection_locator is None:
                self.fresh()
                return True

            try:
                with llfuse.lock_released:
                    self._updating_lock.acquire()
                    if not self.stale():
                        return

                    _logger.debug(
                        "Updating collection %s inode %s to record version %s",
                        self.collection_locator, self.inode, to_record_version)
                    if self.collection is not None:
                        if self.collection.known_past_version(
                                to_record_version):
                            _logger.debug("%s already processed %s",
                                          self.collection_locator,
                                          to_record_version)
                        else:
                            self.collection.update()
                    else:
                        if uuid_pattern.match(self.collection_locator):
                            coll_reader = arvados.collection.Collection(
                                self.collection_locator,
                                self.api,
                                self.api.keep,
                                num_retries=self.num_retries)
                        else:
                            coll_reader = arvados.collection.CollectionReader(
                                self.collection_locator,
                                self.api,
                                self.api.keep,
                                num_retries=self.num_retries)
                        new_collection_record = coll_reader.api_response(
                        ) or {}
                        # If the Collection only exists in Keep, there will be no API
                        # response.  Fill in the fields we need.
                        if 'uuid' not in new_collection_record:
                            new_collection_record[
                                'uuid'] = self.collection_locator
                        if "portable_data_hash" not in new_collection_record:
                            new_collection_record[
                                "portable_data_hash"] = new_collection_record[
                                    "uuid"]
                        if 'manifest_text' not in new_collection_record:
                            new_collection_record[
                                'manifest_text'] = coll_reader.manifest_text()

                        if self.collection_record is None or self.collection_record[
                                "portable_data_hash"] != new_collection_record.get(
                                    "portable_data_hash"):
                            self.new_collection(new_collection_record,
                                                coll_reader)

                        self._manifest_size = len(coll_reader.manifest_text())
                        _logger.debug("%s manifest_size %i", self,
                                      self._manifest_size)
                # end with llfuse.lock_released, re-acquire lock

                self.fresh()
                return True
            finally:
                self._updating_lock.release()
        except arvados.errors.NotFoundError as e:
            _logger.error("Error fetching collection '%s': %s",
                          self.collection_locator, e)
        except arvados.errors.ArgumentError as detail:
            _logger.warning("arv-mount %s: error %s", self.collection_locator,
                            detail)
            if self.collection_record is not None and "manifest_text" in self.collection_record:
                _logger.warning("arv-mount manifest_text is: %s",
                                self.collection_record["manifest_text"])
        except Exception:
            _logger.exception("arv-mount %s: error", self.collection_locator)
            if self.collection_record is not None and "manifest_text" in self.collection_record:
                _logger.error("arv-mount manifest_text is: %s",
                              self.collection_record["manifest_text"])
        self.invalidate()
        return False
Beispiel #15
0
 def want_event_subscribe(self):
     return (uuid_pattern.match(self.collection_locator) is not None)