Exemple #1
0
class Cleaner(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    def run(self):
        self.cache.clean()
Exemple #2
0
class Cleaner(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    def run(self):
        self.cache.clean()
Exemple #3
0
class Cleaner(object):
    def __init__(self, conf, **local_conf):
        self.conf = conf
        self.cache = ImageCache(conf)

    def run(self):
        self.cache.clean()
class Pruner(object):
    def __init__(self, conf, **local_conf):
        self.conf = conf
        self.cache = ImageCache(conf)

    def run(self):
        self.cache.prune()
Exemple #5
0
class Pruner(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    def run(self):
        self.cache.prune()
Exemple #6
0
class Reaper(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    def run(self):
        invalid_grace = int(
            self.options.get('image_cache_invalid_entry_grace_period', 3600))
        self.cache.reap_invalid(grace=invalid_grace)
        self.cache.reap_stalled()
Exemple #7
0
class Reaper(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    def run(self):
        invalid_grace = int(self.options.get(
                            'image_cache_invalid_entry_grace_period',
                            3600))
        self.cache.reap_invalid(grace=invalid_grace)
        self.cache.reap_stalled()
Exemple #8
0
class Prefetcher(object):
    def __init__(self, options):
        self.options = options
        glance.store.create_stores(options)
        self.cache = ImageCache(options)
        registry.configure_registry_client(options)

    def fetch_image_into_cache(self, image_id):
        auth_tok = self.options.get('admin_token')
        ctx = context.RequestContext(is_admin=True,
                                     show_deleted=True,
                                     auth_tok=auth_tok)
        try:
            image_meta = registry.get_image_metadata(ctx, image_id)
            if image_meta['status'] != 'active':
                logger.warn(_("Image '%s' is not active. Not caching."),
                            image_id)
                return False

        except exception.NotFound:
            logger.warn(_("No metadata found for image '%s'"), image_id)
            return False

        image_data, image_size = get_from_backend(image_meta['location'])
        logger.debug(_("Caching image '%s'"), image_id)
        self.cache.cache_image_iter(image_id, image_data)
        return True

    def run(self):

        images = self.cache.get_queued_images()
        if not images:
            logger.debug(_("Nothing to prefetch."))
            return True

        num_images = len(images)
        logger.debug(_("Found %d images to prefetch"), num_images)

        pool = eventlet.GreenPool(num_images)
        results = pool.imap(self.fetch_image_into_cache, images)
        successes = sum([1 for r in results if r is True])
        if successes != num_images:
            logger.error(
                _("Failed to successfully cache all "
                  "images in queue."))
            return False

        logger.info(_("Successfully cached all %d images"), num_images)
        return True
Exemple #9
0
class Prefetcher(object):

    def __init__(self, options):
        self.options = options
        glance.store.create_stores(options)
        self.cache = ImageCache(options)
        registry.configure_registry_client(options)

    def fetch_image_into_cache(self, image_id):
        auth_tok = self.options.get('admin_token')
        ctx = context.RequestContext(is_admin=True, show_deleted=True,
                                     auth_tok=auth_tok)
        try:
            image_meta = registry.get_image_metadata(ctx, image_id)
            if image_meta['status'] != 'active':
                logger.warn(_("Image '%s' is not active. Not caching."),
                            image_id)
                return False

        except exception.NotFound:
            logger.warn(_("No metadata found for image '%s'"), image_id)
            return False

        image_data, image_size = get_from_backend(image_meta['location'])
        logger.debug(_("Caching image '%s'"), image_id)
        self.cache.cache_image_iter(image_id, image_data)
        return True

    def run(self):

        images = self.cache.get_cache_queue()
        if not images:
            logger.debug(_("Nothing to prefetch."))
            return True

        num_images = len(images)
        logger.debug(_("Found %d images to prefetch"), num_images)

        pool = eventlet.GreenPool(num_images)
        results = pool.imap(self.fetch_image_into_cache, images)
        successes = sum([1 for r in results if r is True])
        if successes != num_images:
            logger.error(_("Failed to successfully cache all "
                           "images in queue."))
            return False

        logger.info(_("Successfully cached all %d images"), num_images)
        return True
Exemple #10
0
class Queuer(object):

    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)
        registry.configure_registry_client(options)

    def queue_image(self, image_id):
        auth_tok = self.options.get('admin_token')
        ctx = context.RequestContext(is_admin=True, show_deleted=True,
                                     auth_tok=auth_tok)
        try:
            image_meta = registry.get_image_metadata(ctx, image_id)
            if image_meta['status'] != 'active':
                logger.warn(_("Image '%s' is not active. Not queueing."),
                            image_id)
                return False

        except exception.NotFound:
            logger.warn(_("No metadata found for image '%s'"), image_id)
            return False

        logger.debug(_("Queueing image '%s'"), image_id)
        self.cache.queue_image(image_id)
        return True

    def run(self, images):

        num_images = len(images)
        if num_images == 0:
            logger.debug(_("No images to queue!"))
            return True

        logger.debug(_("Received %d images to queue"), num_images)

        pool = eventlet.GreenPool(num_images)
        results = pool.imap(self.queue_image, images)
        successes = sum([1 for r in results if r is True])
        if successes != num_images:
            logger.error(_("Failed to successfully queue all "
                           "images in queue."))
            return False

        logger.info(_("Successfully queued all %d images"), num_images)
        return True
Exemple #11
0
 def __init__(self, conf, **local_conf):
     self.conf = conf
     self.cache = ImageCache(conf)
Exemple #12
0
 def __init__(self, options):
     self.options = options
     glance.store.create_stores(options)
     self.cache = ImageCache(options)
Exemple #13
0
class Prefetcher(object):
    def __init__(self, options):
        self.options = options
        glance.store.create_stores(options)
        self.cache = ImageCache(options)

    def fetch_image_into_cache(self, image_id):
        ctx = context.RequestContext(is_admin=True, show_deleted=True)
        image_meta = registry.get_image_metadata(
                    self.options, ctx, image_id)
        with self.cache.open(image_meta, "wb") as cache_file:
            chunks = get_from_backend(image_meta['location'],
                                      expected_size=image_meta['size'],
                                      options=self.options)
            for chunk in chunks:
                cache_file.write(chunk)

    def run(self):
        if self.cache.is_currently_prefetching_any_images():
            logger.debug(_("Currently prefetching, going back to sleep..."))
            return

        try:
            image_id = self.cache.pop_prefetch_item()
        except IndexError:
            logger.debug(_("Nothing to prefetch, going back to sleep..."))
            return

        if self.cache.hit(image_id):
            logger.warn(_("Image %s is already in the cache, deleting "
                        "prefetch job and going back to sleep..."), image_id)
            self.cache.delete_queued_prefetch_image(image_id)
            return

        # NOTE(sirp): if someone is already downloading an image that is in
        # the prefetch queue, then go ahead and delete that item and try to
        # prefetch another
        if self.cache.is_image_currently_being_written(image_id):
            logger.warn(_("Image %s is already being cached, deleting "
                        "prefetch job and going back to sleep..."), image_id)
            self.cache.delete_queued_prefetch_image(image_id)
            return

        logger.debug(_("Prefetching '%s'"), image_id)
        self.cache.do_prefetch(image_id)

        try:
            self.fetch_image_into_cache(image_id)
        finally:
            self.cache.delete_prefetching_image(image_id)
Exemple #14
0
 def __init__(self):
     self.cache = ImageCache()
Exemple #15
0
 def __init__(self, options):
     self.options = options
     self.cache = ImageCache(options)
Exemple #16
0
class Pruner(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    @property
    def max_size(self):
        default = 1 * 1024 * 1024 * 1024  # 1 GB
        return config.get_option(
            self.options, 'image_cache_max_size_bytes',
            type='int', default=default)

    @property
    def percent_extra_to_free(self):
        return config.get_option(
            self.options, 'image_cache_percent_extra_to_free',
            type='float', default=0.05)

    def run(self):
        self.prune_cache()

    def prune_cache(self):
        """Prune the cache using an LRU strategy"""

        # NOTE(sirp): 'Recency' is determined via the filesystem, first using
        # atime (access time) and falling back to mtime (modified time).
        #
        # It has become more common to disable access-time updates by setting
        # the `noatime` option for the filesystem. `noatime` is NOT compatible
        # with this method.
        #
        # If `noatime` needs to be supported, we will need to persist access
        # times elsewhere (either as a separate file, in the DB, or as
        # an xattr).
        def get_stats():
            stats = []
            for path in self.cache.get_all_regular_files(self.cache.path):
                file_info = os.stat(path)
                stats.append((file_info[stat.ST_ATIME],  # access time
                              file_info[stat.ST_MTIME],  # modification time
                              file_info[stat.ST_SIZE],   # size in bytes
                              path))                     # absolute path
            return stats

        def prune_lru(stats, to_free):
            # Sort older access and modified times to the back
            stats.sort(reverse=True)

            freed = 0
            while to_free > 0:
                atime, mtime, size, path = stats.pop()
                logger.debug("deleting '%(path)s' to free %(size)d B",
                             locals())
                os.unlink(path)
                to_free -= size
                freed += size

            return freed

        stats = get_stats()

        # Check for overage
        cur_size = sum(s[2] for s in stats)
        max_size = self.max_size
        logger.debug("cur_size=%(cur_size)d B max_size=%(max_size)d B",
                     locals())
        if cur_size <= max_size:
            logger.debug("cache has free space, skipping prune...")
            return

        overage = cur_size - max_size
        extra = max_size * self.percent_extra_to_free
        to_free = overage + extra
        logger.debug("overage=%(overage)d B extra=%(extra)d B"
                     " total=%(to_free)d B", locals())

        freed = prune_lru(stats, to_free)
        logger.debug("finished pruning, freed %(freed)d bytes", locals())
Exemple #17
0
 def __init__(self, options):
     self.options = options
     self.cache = ImageCache(options)
Exemple #18
0
class Pruner(object):
    def __init__(self, options):
        self.options = options
        self.cache = ImageCache(options)

    @property
    def max_size(self):
        default = 1 * 1024 * 1024 * 1024  # 1 GB
        return config.get_option(self.options,
                                 'image_cache_max_size_bytes',
                                 type='int',
                                 default=default)

    @property
    def percent_extra_to_free(self):
        return config.get_option(self.options,
                                 'image_cache_percent_extra_to_free',
                                 type='float',
                                 default=0.05)

    def run(self):
        self.prune_cache()

    def prune_cache(self):
        """Prune the cache using an LRU strategy"""

        # NOTE(sirp): 'Recency' is determined via the filesystem, first using
        # atime (access time) and falling back to mtime (modified time).
        #
        # It has become more common to disable access-time updates by setting
        # the `noatime` option for the filesystem. `noatime` is NOT compatible
        # with this method.
        #
        # If `noatime` needs to be supported, we will need to persist access
        # times elsewhere (either as a separate file, in the DB, or as
        # an xattr).
        def get_stats():
            stats = []
            for path in self.cache.get_all_regular_files(self.cache.path):
                file_info = os.stat(path)
                stats.append((
                    file_info[stat.ST_ATIME],  # access time
                    file_info[stat.ST_MTIME],  # modification time
                    file_info[stat.ST_SIZE],  # size in bytes
                    path))  # absolute path
            return stats

        def prune_lru(stats, to_free):
            # Sort older access and modified times to the back
            stats.sort(reverse=True)

            freed = 0
            while to_free > 0:
                atime, mtime, size, path = stats.pop()
                logger.debug(_("deleting '%(path)s' to free %(size)d B"),
                             locals())
                os.unlink(path)
                to_free -= size
                freed += size

            return freed

        stats = get_stats()

        # Check for overage
        cur_size = sum(s[2] for s in stats)
        max_size = self.max_size
        logger.debug(_("cur_size=%(cur_size)d B max_size=%(max_size)d B"),
                     locals())
        if cur_size <= max_size:
            logger.debug(_("cache has free space, skipping prune..."))
            return

        overage = cur_size - max_size
        extra = max_size * self.percent_extra_to_free
        to_free = overage + extra
        logger.debug(
            _("overage=%(overage)d B extra=%(extra)d B"
              " total=%(to_free)d B"), locals())

        freed = prune_lru(stats, to_free)
        logger.debug(_("finished pruning, freed %(freed)d bytes"), locals())
Exemple #19
0
 def __init__(self, options):
     self.options = options
     glance.store.create_stores(options)
     self.cache = ImageCache(options)
     registry.configure_registry_client(options)
 def __init__(self, conf, **local_conf):
     self.conf = conf
     glance.store.create_stores(conf)
     self.cache = ImageCache(conf)
     registry.configure_registry_client(conf)
     registry.configure_registry_admin_creds(conf)
Exemple #21
0
 def __init__(self, conf, **local_conf):
     self.conf = conf
     self.cache = ImageCache(conf)
     registry.configure_registry_client(conf)
     registry.configure_registry_admin_creds(conf)
Exemple #22
0
 def __init__(self, options):
     self.options = options
     self.cache = ImageCache(options)
     registry.configure_registry_client(options)
Exemple #23
0
 def __init__(self, options):
     self.options = options
     self.cache = ImageCache(options)
     registry.configure_registry_client(options)
Exemple #24
0
 def __init__(self, conf, **local_conf):
     self.conf = conf
     self.cache = ImageCache(conf)
     registry.configure_registry_client(conf)
 def __init__(self, conf, **local_conf):
     self.conf = conf
     self.cache = ImageCache(conf)
Exemple #26
0
 def __init__(self, options):
     self.options = options
     glance.store.create_stores(options)
     self.cache = ImageCache(options)
     registry.configure_registry_client(options)
Exemple #27
0
 def __init__(self, conf, **local_conf):
     self.conf = conf
     glance.store.create_stores(conf)
     self.cache = ImageCache(conf)
     registry.configure_registry_client(conf)
     registry.configure_registry_admin_creds(conf)