Example #1
0
    def _cleanup(self, pool):
        now = time.time()
        cleanup_file = os.path.join(CONF.scrubber_datadir, ".cleanup")
        if not os.path.exists(cleanup_file):
            self._update_cleanup_file(cleanup_file, now)
            return

        last_cleanup_time = self._read_cleanup_file(cleanup_file)
        cleanup_time = last_cleanup_time + CONF.cleanup_scrubber_time
        if cleanup_time > now:
            return

        LOG.info(
            _("Getting images deleted before "
              "%s") % CONF.cleanup_scrubber_time)
        self._update_cleanup_file(cleanup_file, now)

        delete_jobs = self._get_delete_jobs(self.db_queue, False)
        if not delete_jobs:
            return

        for image_id, jobs in delete_jobs.iteritems():
            with lockutils.lock("scrubber-%s" % image_id,
                                lock_file_prefix='glance-',
                                external=True):
                if not self.file_queue.has_image(image_id):
                    # NOTE(zhiyan): scrubber should not cleanup this image
                    # since a queue file be created for this 'pending_delete'
                    # image concurrently before the code get lock and
                    # reach here. The checking only be worth if glance-api and
                    # glance-scrubber service be deployed on a same host.
                    self._scrub_image(pool, image_id, jobs)
Example #2
0
    def _cleanup(self, pool):
        now = time.time()
        cleanup_file = os.path.join(CONF.scrubber_datadir, ".cleanup")
        if not os.path.exists(cleanup_file):
            self._update_cleanup_file(cleanup_file, now)
            return

        last_cleanup_time = self._read_cleanup_file(cleanup_file)
        cleanup_time = last_cleanup_time + CONF.cleanup_scrubber_time
        if cleanup_time > now:
            return

        LOG.info(_("Getting images deleted before "
                   "%s") % CONF.cleanup_scrubber_time)
        self._update_cleanup_file(cleanup_file, now)

        delete_jobs = self._get_delete_jobs(self.db_queue, False)
        if not delete_jobs:
            return

        for image_id, jobs in delete_jobs.iteritems():
            with lockutils.lock("scrubber-%s" % image_id,
                                lock_file_prefix='glance-', external=True):
                if not self.file_queue.has_image(image_id):
                    # NOTE(zhiyan): scrubber should not cleanup this image
                    # since a queue file be created for this 'pending_delete'
                    # image concurrently before the code get lock and
                    # reach here. The checking only be worth if glance-api and
                    # glance-scrubber service be deployed on a same host.
                    self._scrub_image(pool, image_id, jobs)
Example #3
0
    def add_location(self, image_id, location, user_context=None):
        """Adding image location to scrub queue.

        :param image_id: The opaque image identifier
        :param location: The opaque image location
        :param user_context: The user's request context

        :retval A boolean value to indicate success or not
        """
        if user_context is not None:
            registry_client = registry.get_registry_client(user_context)
        else:
            registry_client = self.registry

        with lockutils.lock("scrubber-%s" % image_id,
                            lock_file_prefix='glance-',
                            external=True):

            # NOTE(zhiyan): make sure scrubber does not cleanup
            # 'pending_delete' images concurrently before the code
            # get lock and reach here.
            try:
                image = registry_client.get_image(image_id)
                if image['status'] == 'deleted':
                    return True
            except exception.NotFound as e:
                LOG.warn(_LW("Failed to find image to delete: %s"),
                         utils.exception_to_str(e))
                return False

            loc_id = location.get('id', '-')
            if self.metadata_encryption_key:
                uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
                                            location['url'], 64)
            else:
                uri = location['url']
            delete_time = time.time() + self.scrub_time
            file_path = os.path.join(self.scrubber_datadir, str(image_id))

            if os.path.exists(file_path):
                # Append the uri of location to the queue file
                with open(file_path, 'a') as f:
                    f.write('\n')
                    f.write('\n'.join(
                        [str(loc_id), uri,
                         str(int(delete_time))]))
            else:
                # NOTE(zhiyan): Protect the file before we write any data.
                open(file_path, 'w').close()
                os.chmod(file_path, 0o600)
                with open(file_path, 'w') as f:
                    f.write('\n'.join(
                        [str(loc_id), uri,
                         str(int(delete_time))]))
            os.utime(file_path, (delete_time, delete_time))

            return True
Example #4
0
    def add_location(self, image_id, location, user_context=None):
        """Adding image location to scrub queue.

        :param image_id: The opaque image identifier
        :param location: The opaque image location
        :param user_context: The user's request context

        :retval A boolean value to indicate success or not
        """
        if user_context is not None:
            registry_client = registry.get_registry_client(user_context)
        else:
            registry_client = self.registry

        with lockutils.lock("scrubber-%s" % image_id,
                            lock_file_prefix='glance-', external=True):

            # NOTE(zhiyan): make sure scrubber does not cleanup
            # 'pending_delete' images concurrently before the code
            # get lock and reach here.
            try:
                image = registry_client.get_image(image_id)
                if image['status'] == 'deleted':
                    return True
            except exception.NotFound as e:
                LOG.warn(_LW("Failed to find image to delete: %s"),
                         utils.exception_to_str(e))
                return False

            loc_id = location.get('id', '-')
            if self.metadata_encryption_key:
                uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
                                            location['url'], 64)
            else:
                uri = location['url']
            delete_time = time.time() + self.scrub_time
            file_path = os.path.join(self.scrubber_datadir, str(image_id))

            if os.path.exists(file_path):
                # Append the uri of location to the queue file
                with open(file_path, 'a') as f:
                    f.write('\n')
                    f.write('\n'.join([str(loc_id),
                                       uri,
                                       str(int(delete_time))]))
            else:
                # NOTE(zhiyan): Protect the file before we write any data.
                open(file_path, 'w').close()
                os.chmod(file_path, 0o600)
                with open(file_path, 'w') as f:
                    f.write('\n'.join([str(loc_id),
                                       uri,
                                       str(int(delete_time))]))
            os.utime(file_path, (delete_time, delete_time))

            return True
Example #5
0
    def _walk_all_locations(self, remove=False):
        """Returns a list of image id and location tuple from scrub queue.

        :param remove: Whether remove location from queue or not after walk

        :retval a list of image id, location id and uri tuple from scrub queue
        """
        if not os.path.exists(self.scrubber_datadir):
            LOG.warn(
                _LW("%s directory does not exist.") % self.scrubber_datadir)
            return []

        ret = []
        for root, dirs, files in os.walk(self.scrubber_datadir):
            for image_id in files:
                if not utils.is_uuid_like(image_id):
                    continue
                with lockutils.lock("scrubber-%s" % image_id,
                                    lock_file_prefix='glance-',
                                    external=True):
                    file_path = os.path.join(self.scrubber_datadir, image_id)
                    records = self._read_queue_file(file_path)
                    loc_ids, uris, delete_times = records

                    remove_record_idxs = []
                    skipped = False
                    for (record_idx, delete_time) in enumerate(delete_times):
                        if delete_time > time.time():
                            skipped = True
                            continue
                        else:
                            ret.append((image_id, loc_ids[record_idx],
                                        uris[record_idx]))
                            remove_record_idxs.append(record_idx)

                    if remove:
                        if skipped:
                            # NOTE(zhiyan): remove location records from
                            # the queue file.
                            self._update_queue_file(file_path,
                                                    remove_record_idxs)
                        else:
                            utils.safe_remove(file_path)
        return ret
Example #6
0
    def _walk_all_locations(self, remove=False):
        """Returns a list of image id and location tuple from scrub queue.

        :param remove: Whether remove location from queue or not after walk

        :retval a list of image id, location id and uri tuple from scrub queue
        """
        if not os.path.exists(self.scrubber_datadir):
            LOG.info(_LI("%s directory does not exist.") %
                     self.scrubber_datadir)
            return []

        ret = []
        for root, dirs, files in os.walk(self.scrubber_datadir):
            for image_id in files:
                if not utils.is_uuid_like(image_id):
                    continue
                with lockutils.lock("scrubber-%s" % image_id,
                                    lock_file_prefix='glance-', external=True):
                    file_path = os.path.join(self.scrubber_datadir, image_id)
                    records = self._read_queue_file(file_path)
                    loc_ids, uris, delete_times = records

                    remove_record_idxs = []
                    skipped = False
                    for (record_idx, delete_time) in enumerate(delete_times):
                        if delete_time > time.time():
                            skipped = True
                            continue
                        else:
                            ret.append((image_id,
                                        loc_ids[record_idx],
                                        uris[record_idx]))
                            remove_record_idxs.append(record_idx)

                    if remove:
                        if skipped:
                            # NOTE(zhiyan): remove location records from
                            # the queue file.
                            self._update_queue_file(file_path,
                                                    remove_record_idxs)
                        else:
                            utils.safe_remove(file_path)
        return ret
Example #7
0
    def add_location(self, image_id, uri):
        """Adding image location to scrub queue.

        :param image_id: The opaque image identifier
        :param uri: The opaque image location uri
        """
        with lockutils.lock("scrubber-%s" % image_id,
                            lock_file_prefix='glance-',
                            external=True):

            # NOTE(zhiyan): make sure scrubber does not cleanup
            # 'pending_delete' images concurrently before the code
            # get lock and reach here.
            try:
                image = self.registry.get_image(image_id)
                if image['status'] == 'deleted':
                    return
            except exception.NotFound as e:
                LOG.error(
                    _("Failed to find image to delete: "
                      "%(e)s") % locals())
                return

            delete_time = time.time() + self.scrub_time
            file_path = os.path.join(self.scrubber_datadir, str(image_id))

            if self.metadata_encryption_key is not None:
                uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, uri,
                                            64)

            if os.path.exists(file_path):
                # Append the uri of location to the queue file
                with open(file_path, 'a') as f:
                    f.write('\n')
                    f.write('\n'.join([uri, str(int(delete_time))]))
            else:
                # NOTE(zhiyan): Protect the file before we write any data.
                open(file_path, 'w').close()
                os.chmod(file_path, 0o600)
                with open(file_path, 'w') as f:
                    f.write('\n'.join([uri, str(int(delete_time))]))
            os.utime(file_path, (delete_time, delete_time))
Example #8
0
    def add_location(self, image_id, uri):
        """Adding image location to scrub queue.

        :param image_id: The opaque image identifier
        :param uri: The opaque image location uri
        """
        with lockutils.lock("scrubber-%s" % image_id,
                            lock_file_prefix='glance-', external=True):

            # NOTE(zhiyan): make sure scrubber does not cleanup
            # 'pending_delete' images concurrently before the code
            # get lock and reach here.
            try:
                image = self.registry.get_image(image_id)
                if image['status'] == 'deleted':
                    return
            except exception.NotFound as e:
                LOG.error(_("Failed to find image to delete: "
                            "%(e)s") % locals())
                return

            delete_time = time.time() + self.scrub_time
            file_path = os.path.join(self.scrubber_datadir, str(image_id))

            if self.metadata_encryption_key is not None:
                uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
                                            uri, 64)

            if os.path.exists(file_path):
                # Append the uri of location to the queue file
                with open(file_path, 'a') as f:
                    f.write('\n')
                    f.write('\n'.join([uri, str(int(delete_time))]))
            else:
                # NOTE(zhiyan): Protect the file before we write any data.
                open(file_path, 'w').close()
                os.chmod(file_path, 0o600)
                with open(file_path, 'w') as f:
                    f.write('\n'.join([uri, str(int(delete_time))]))
            os.utime(file_path, (delete_time, delete_time))