Esempio n. 1
0
    def _delete_image_location_from_backend(self, image_id, loc_id, uri,
                                            backend=None):
        try:
            LOG.debug("Scrubbing image %s from a location.", image_id)
            try:
                if CONF.enabled_backends:
                    self.store_api.delete(uri, backend, self.admin_context)
                else:
                    self.store_api.delete_from_backend(uri, self.admin_context)
            except store_exceptions.NotFound:
                LOG.info(_LI("Image location for image '%s' not found in "
                             "backend; Marking image location deleted in "
                             "db."), image_id)

            if loc_id != '-':
                db_api.get_api().image_location_delete(self.admin_context,
                                                       image_id,
                                                       int(loc_id),
                                                       'deleted')
            LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
        except Exception as e:
            LOG.error(_LE("Unable to scrub image %(id)s from a location. "
                          "Reason: %(exc)s ") %
                      {'id': image_id,
                       'exc': encodeutils.exception_to_unicode(e)})
            raise
Esempio n. 2
0
def safe_delete_from_backend(context, image_id, location):
    """
    Given a location, delete an image from the store and
    update location status to db.

    This function try to handle all known exceptions which might be raised
    by those calls on store and DB modules in its implementation.

    :param context: The request context
    :param image_id: The image identifier
    :param location: The image location entry
    """

    try:
        ret = store_api.delete_from_backend(location['url'], context=context)
        location['status'] = 'deleted'
        if 'id' in location:
            db_api.get_api().image_location_delete(context, image_id,
                                                   location['id'], 'deleted')
        return ret
    except store_api.NotFound:
        msg = _LW('Failed to delete image %s in store from URI') % image_id
        LOG.warn(msg)
    except store_api.StoreDeleteNotSupported as e:
        LOG.warn(encodeutils.exception_to_unicode(e))
    except store_api.UnsupportedBackend:
        exc_type = sys.exc_info()[0].__name__
        msg = (_LE('Failed to delete image %(image_id)s from store: %(exc)s') %
               dict(image_id=image_id, exc=exc_type))
        LOG.error(msg)
Esempio n. 3
0
    def _scrub_image(self, image_id, delete_jobs):
        if len(delete_jobs) == 0:
            return

        LOG.info(_LI("Scrubbing image %(id)s from %(count)d locations."),
                 {'id': image_id, 'count': len(delete_jobs)})

        success = True
        if CONF.enabled_backends:
            for img_id, loc_id, uri, backend in delete_jobs:
                try:
                    self._delete_image_location_from_backend(img_id, loc_id,
                                                             uri,
                                                             backend=backend)
                except Exception:
                    success = False
        else:
            for img_id, loc_id, uri in delete_jobs:
                try:
                    self._delete_image_location_from_backend(img_id, loc_id,
                                                             uri)
                except Exception:
                    success = False

        if success:
            image = db_api.get_api().image_get(self.admin_context, image_id)
            if image['status'] == 'pending_delete':
                db_api.get_api().image_update(self.admin_context, image_id,
                                              {'status': 'deleted'})
            LOG.info(_LI("Image %s has been scrubbed successfully"), image_id)
        else:
            LOG.warn(_LW("One or more image locations couldn't be scrubbed "
                         "from backend. Leaving image '%s' in 'pending_delete'"
                         " status") % image_id)
Esempio n. 4
0
 def _get_pending_delete_image(self, image_id):
     # In Glance V2, there is no way to get the 'pending_delete' image from
     # API. So we get the image from db here for testing.
     # Clean the session cache first to avoid connecting to the old db data.
     db_api.get_api()._FACADE = None
     image = db_api.get_api().image_get(self.admin_context, image_id)
     return image
Esempio n. 5
0
def schedule_delayed_delete_from_backend(context, image_id, location):
    """
    Given a location, schedule the deletion of an image location and
    update location status to db.

    :param context: The request context
    :param image_id: The image identifier
    :param location: The image location entry
    """

    db_queue = scrubber.get_scrub_queue()

    if not CONF.use_user_token:
        context = None

    ret = db_queue.add_location(image_id, location)
    if ret:
        location['status'] = 'pending_delete'
        if 'id' in location:
            # NOTE(zhiyan): New added image location entry will has no 'id'
            # field since it has not been saved to DB.
            db_api.get_api().image_location_delete(context, image_id,
                                                   location['id'],
                                                   'pending_delete')
        else:
            db_api.get_api().image_location_add(context, image_id, location)

    return ret
Esempio n. 6
0
    def _delete_image_location_from_backend(self, image_id, loc_id, uri):
        if CONF.metadata_encryption_key:
            uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)

        try:
            LOG.debug("Deleting URI from image %s." % image_id)
            self.store_api.delete_from_backend(self.admin_context, uri)
            if loc_id != "-":
                db_api.get_api().image_location_delete(self.admin_context, image_id, int(loc_id), "deleted")
            LOG.info(_LI("Image %s has been deleted.") % image_id)
        except Exception:
            LOG.warn(_LW("Unable to delete URI from image %s.") % image_id)
Esempio n. 7
0
    def add_location(self, image_id, location, user_context=None):
        """Adding image location to scrub queue.

        :param image_id: The opaque image identifier
        :param location: The opaque image location
        :param user_context: The user's request context

        :retval A boolean value to indicate success or not
        """
        loc_id = location.get("id")
        if loc_id:
            db_api.get_api().image_location_delete(self.admin_context, image_id, loc_id, "pending_delete")
            return True
        else:
            return False
Esempio n. 8
0
    def add_location(self, image_id, location):
        """Adding image location to scrub queue.

        :param image_id: The opaque image identifier
        :param location: The opaque image location

        :retval A boolean value to indicate success or not
        """
        loc_id = location.get('id')
        if loc_id:
            db_api.get_api().image_location_delete(self.admin_context,
                                                   image_id, loc_id,
                                                   'pending_delete')
            return True
        else:
            return False
Esempio n. 9
0
    def _get_images_page(self, marker):
        filters = {'deleted': True,
                   'status': 'pending_delete'}

        return db_api.get_api().image_get_all(self.admin_context,
                                              filters=filters,
                                              marker=marker,
                                              limit=REASONABLE_DB_PAGE_SIZE)
Esempio n. 10
0
    def test_unwrap(self):
        CONF.set_override('use_tpool', True)
        CONF.set_override('data_api', 'glance.tests.functional.db.'
                          'test_db_api')

        dbapi = db_api.get_api()
        self.assertEqual(importutils.import_module(CONF.data_api),
                         dbapi.unwrap())
Esempio n. 11
0
    def _walk_all_locations(self, remove=False):
        """Returns a list of image id and location tuple from scrub queue.

        :param remove: Whether remove location from queue or not after walk

        :retval a list of image id, location id and uri tuple from scrub queue
        """
        filters = {'deleted': True,
                   'is_public': 'none',
                   'status': 'pending_delete'}
        ret = []
        for image in self.registry.get_images_detailed(filters=filters):
            deleted_at = image.get('deleted_at')
            if not deleted_at:
                continue

            # NOTE: Strip off microseconds which may occur after the last '.,'
            # Example: 2012-07-07T19:14:34.974216
            date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
            delete_time = calendar.timegm(time.strptime(date_str,
                                                        "%Y-%m-%dT%H:%M:%S"))

            if delete_time + self.scrub_time > time.time():
                continue

            for loc in image['location_data']:
                if loc['status'] != 'pending_delete':
                    continue

                if self.metadata_encryption_key:
                    uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
                                                loc['url'], 64)
                else:
                    uri = loc['url']

                ret.append((image['id'], loc['id'], uri))

                if remove:
                    db_api.get_api().image_location_delete(self.admin_context,
                                                           image['id'],
                                                           loc['id'],
                                                           'deleted')
                    self.registry.update_image(image['id'],
                                               {'status': 'deleted'})
        return ret
Esempio n. 12
0
    def test_thread_pool(self):
        CONF.set_override('use_tpool', True)
        CONF.set_override('data_api', 'glance.tests.functional.db.'
                          'test_db_api')
        dbapi = db_api.get_api()

        from eventlet import tpool
        tpool.execute = Mock()

        dbapi.method_for_test_1(1, 2, kwarg='arg')
        tpool.execute.assert_called_with(method_for_test_1, 1, 2, kwarg='arg')
Esempio n. 13
0
    def has_image(self, image_id):
        """Returns whether the queue contains an image or not.

        :param image_id: The opaque image identifier

        :returns: a boolean value to inform including or not
        """
        try:
            image = db_api.get_api().image_get(self.admin_context, image_id)
            return image['status'] == 'pending_delete'
        except exception.NotFound:
            return False
Esempio n. 14
0
    def test_thread_pool(self):
        module = importutils.import_module("glance.tests.functional.db." "test_db_api")
        CONF.set_override("use_tpool", True)
        CONF.set_override("data_api", "glance.tests.functional.db." "test_db_api")
        dbapi = db_api.get_api()

        from eventlet import tpool

        tpool.execute = Mock()

        dbapi.method_for_test_1(1, 2, kwarg="arg")
        tpool.execute.assert_called_with(method_for_test_1, 1, 2, kwarg="arg")
Esempio n. 15
0
    def _scrub_image(self, image_id, delete_jobs):
        if len(delete_jobs) == 0:
            return

        LOG.info(_LI("Scrubbing image %(id)s from %(count)d locations."), {
            'id': image_id,
            'count': len(delete_jobs)
        })

        success = True
        if CONF.enabled_backends:
            for img_id, loc_id, uri, backend in delete_jobs:
                try:
                    self._delete_image_location_from_backend(img_id,
                                                             loc_id,
                                                             uri,
                                                             backend=backend)
                except Exception:
                    success = False
        else:
            for img_id, loc_id, uri in delete_jobs:
                try:
                    self._delete_image_location_from_backend(
                        img_id, loc_id, uri)
                except Exception:
                    success = False

        if success:
            image = db_api.get_api().image_get(self.admin_context, image_id)
            if image['status'] == 'pending_delete':
                db_api.get_api().image_update(self.admin_context, image_id,
                                              {'status': 'deleted'})
            LOG.info(_LI("Image %s has been scrubbed successfully"), image_id)
        else:
            LOG.warn(
                _LW("One or more image locations couldn't be scrubbed "
                    "from backend. Leaving image '%s' in 'pending_delete'"
                    " status") % image_id)
Esempio n. 16
0
def safe_delete_from_backend(context, image_id, location):
    """
    Given a location, delete an image from the store and
    update location status to db.

    This function try to handle all known exceptions which might be raised
    by those calls on store and DB modules in its implementation.

    :param context: The request context
    :param image_id: The image identifier
    :param location: The image location entry
    """

    try:
        if CONF.enabled_backends:
            backend = location['metadata'].get('store')
            ret = store_api.delete(location['url'],
                                   backend,
                                   context=context)
        else:
            ret = store_api.delete_from_backend(location['url'],
                                                context=context)

        location['status'] = 'deleted'
        if 'id' in location:
            db_api.get_api().image_location_delete(context, image_id,
                                                   location['id'], 'deleted')
        return ret
    except store_api.NotFound:
        msg = _LW('Failed to delete image %s in store from URI') % image_id
        LOG.warn(msg)
    except store_api.StoreDeleteNotSupported as e:
        LOG.warn(encodeutils.exception_to_unicode(e))
    except store_api.UnsupportedBackend:
        exc_type = sys.exc_info()[0].__name__
        msg = (_LE('Failed to delete image %(image_id)s from store: %(exc)s') %
               dict(image_id=image_id, exc=exc_type))
        LOG.error(msg)
Esempio n. 17
0
    def _walk_all_locations(self, remove=False):
        """Returns a list of image id and location tuple from scrub queue.

        :param remove: Whether remove location from queue or not after walk

        :retval a list of image id, location id and uri tuple from scrub queue
        """
        filters = {"deleted": True, "is_public": "none", "status": "pending_delete"}
        ret = []
        for image in self.registry.get_images_detailed(filters=filters):
            deleted_at = image.get("deleted_at")
            if not deleted_at:
                continue

            # NOTE: Strip off microseconds which may occur after the last '.,'
            # Example: 2012-07-07T19:14:34.974216
            date_str = deleted_at.rsplit(".", 1)[0].rsplit(",", 1)[0]
            delete_time = calendar.timegm(time.strptime(date_str, "%Y-%m-%dT%H:%M:%S"))

            if delete_time + self.scrub_time > time.time():
                continue

            for loc in image["location_data"]:
                if loc["status"] != "pending_delete":
                    continue

                if self.metadata_encryption_key:
                    uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64)
                else:
                    uri = loc["url"]

                ret.append((image["id"], loc["id"], uri))

                if remove:
                    db_api.get_api().image_location_delete(self.admin_context, image["id"], loc["id"], "deleted")
                    self.registry.update_image(image["id"], {"status": "deleted"})
        return ret
Esempio n. 18
0
    def _delete_image_location_from_backend(self, image_id, loc_id, uri):
        try:
            LOG.debug("Scrubbing image %s from a location.", image_id)
            try:
                self.store_api.delete_from_backend(uri, self.admin_context)
            except store_exceptions.NotFound:
                LOG.info(
                    _LI("Image location for image '%s' not found in "
                        "backend; Marking image location deleted in "
                        "db."), image_id)

            if loc_id != '-':
                db_api.get_api().image_location_delete(self.admin_context,
                                                       image_id, int(loc_id),
                                                       'deleted')
            LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
        except Exception as e:
            LOG.error(
                _LE("Unable to scrub image %(id)s from a location. "
                    "Reason: %(exc)s ") % {
                        'id': image_id,
                        'exc': encodeutils.exception_to_unicode(e)
                    })
            raise
Esempio n. 19
0
    def __init__(self,
                 store,
                 store_location,
                 context=None,
                 allow_reauth=False):
        # no context - no party
        if context is None:
            reason = _("Multi-tenant Swift storage requires a user context.")
            raise exceptions.BadStoreConfiguration(store_name="swift",
                                                   reason=reason)

        api = glance_db.get_api()
        self.image = api.image_get(context, store_location.obj)

        super(MultiTenantConnectionManager,
              self).__init__(store, store_location, context, allow_reauth)
Esempio n. 20
0
 def wait_for_scrub(self, image_id):
     """
     NOTE(jkoelker) The build servers sometimes take longer than 15 seconds
     to scrub. Give it up to 5 min, checking checking every 15 seconds.
     When/if it flips to deleted, bail immediately.
     """
     wait_for = 300  # seconds
     check_every = 15  # seconds
     for _ in range(wait_for // check_every):
         time.sleep(check_every)
         image = db_api.get_api().image_get(self.admin_context, image_id)
         if (image['status'] == 'deleted' and image['deleted'] == True):
             break
         else:
             continue
     else:
         self.fail('image was never scrubbed')
Esempio n. 21
0
 def wait_for_scrub(self, image_id):
     """
     NOTE(jkoelker) The build servers sometimes take longer than 15 seconds
     to scrub. Give it up to 5 min, checking checking every 15 seconds.
     When/if it flips to deleted, bail immediately.
     """
     wait_for = 300    # seconds
     check_every = 15  # seconds
     for _ in range(wait_for // check_every):
         time.sleep(check_every)
         image = db_api.get_api().image_get(self.admin_context, image_id)
         if (image['status'] == 'deleted' and
                 image['deleted'] == True):
             break
         else:
             continue
     else:
         self.fail('image was never scrubbed')
Esempio n. 22
0
 def revert_image_status(self, image_id):
     db_api.get_api().image_restore(self.admin_context, image_id)
Esempio n. 23
0
 def test_unwrap_dbapi_when_db_pool_is_disabled(self):
     CONF.set_override('use_tpool', True)
     dbapi = db_api.get_api()
     self.assertEqual(importutils.import_module(CONF.data_api),
                      glance.db.unwrap(dbapi))
Esempio n. 24
0
 def test_get_dbapi_when_db_pool_is_disabled(self):
     CONF.set_override('use_tpool', False)
     dbapi = db_api.get_api()
     self.assertFalse(isinstance(dbapi, db_api.ThreadPoolWrapper))
     self.assertEqual(importutils.import_module(CONF.data_api), dbapi)
Esempio n. 25
0
 def test_get_dbapi_when_db_pool_is_enabled(self):
     CONF.set_override('use_tpool', True)
     dbapi = db_api.get_api()
     self.assertTrue(isinstance(dbapi, db_api.ThreadPoolWrapper))
Esempio n. 26
0
 def revert_image_status(self, image_id):
     db_api.get_api().image_restore(self.admin_context, image_id)