def test_http_schedule_delete_swallows_error(self): uri = {"url": "https://netloc/path/to/file.tar.gz"} ctx = context.RequestContext() stub_out_registry_image_update(self.stubs) try: safe_delete_from_backend(ctx, 'image_id', uri) except exception.StoreDeleteNotSupported: self.fail('StoreDeleteNotSupported should be swallowed')
def test_initiate_delete(self): req = unit_test_utils.get_fake_request() location = {"url": "file://foo/bar", "metadata": {}, "status": "active"} id = unit_test_utils.UUID1 self.mox.StubOutWithMock(store_utils, "safe_delete_from_backend") store_utils.safe_delete_from_backend(req.context, id, location) self.mox.ReplayAll() upload_utils.initiate_deletion(req, location, id) self.mox.VerifyAll()
def test_initiate_delete(self): req = unit_test_utils.get_fake_request() location = { "url": "file://foo/bar", "metadata": {}, "status": "active" } id = unit_test_utils.UUID1 self.mox.StubOutWithMock(store_utils, "safe_delete_from_backend") store_utils.safe_delete_from_backend(req.context, id, location) self.mox.ReplayAll() upload_utils.initiate_deletion(req, location, id) self.mox.VerifyAll()
def _quota_exceeded_size(self, quota, data, deleted=True, size=None): self.config(user_storage_quota=quota) context = FakeContext() db_api = unit_test_utils.FakeDB() store_api = unit_test_utils.FakeStoreAPI() store = unit_test_utils.FakeStoreUtils(store_api) base_image = FakeImage() base_image.image_id = 'id' image = glance.quota.ImageProxy(base_image, context, db_api, store) if deleted: with patch.object(store_utils, 'safe_delete_from_backend'): store_utils.safe_delete_from_backend(context, image.image_id, base_image.locations[0]) self.assertRaises(exception.StorageQuotaFull, image.set_data, data, size=size)
def _quota_exceeded_size(self, quota, data, deleted=True, size=None): self.config(user_storage_quota=quota) context = FakeContext() db_api = unit_test_utils.FakeDB() store_api = unit_test_utils.FakeStoreAPI() store = unit_test_utils.FakeStoreUtils(store_api) base_image = FakeImage() base_image.image_id = 'id' image = glance.quota.ImageProxy(base_image, context, db_api, store) if deleted: with patch.object(store_utils, 'safe_delete_from_backend'): store_utils.safe_delete_from_backend( context, image.image_id, base_image.locations[0]) self.assertRaises(exception.StorageQuotaFull, image.set_data, data, size=size)
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] db_api = glance.db.get_api() image_size = image_meta.get('size') try: remaining = glance.api.common.check_quota(req.context, image_size, db_api, image_id=image_id) if remaining is not None: image_data = utils.LimitingReader(image_data, remaining) (uri, size, checksum, location_metadata) = store_api.store_add_to_backend( image_meta['id'], utils.CooperativeReader(image_data), image_meta['size'], store) location_data = { 'url': uri, 'metadata': location_metadata, 'status': 'active' } try: # recheck the quota in case there were simultaneous uploads that # did not provide the size glance.api.common.check_quota(req.context, size, db_api, image_id=image_id) except exception.StorageQuotaFull: with excutils.save_and_reraise_exception(): LOG.info( _('Cleaning up %s after exceeding ' 'the quota') % image_id) store_utils.safe_delete_from_backend(req.context, image_meta['id'], location_data) def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = (_("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % { 'attr': attr, 'supplied': supplied, 'actual': actual }) LOG.error(msg) safe_kill(req, image_id) initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug( "Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d", { 'image_id': image_id, 'checksum': checksum, 'size': size }) update_data = {'checksum': checksum, 'size': size} try: image_meta = registry.update_image_metadata( req.context, image_id, update_data) except exception.NotFound as e: msg = _("Image %s could not be found after upload. The image may " "have been deleted during the upload.") % image_id LOG.info(msg) # NOTE(jculp): we need to clean up the datastore if an image # resource is deleted while the image data is being uploaded # # We get "location_data" from above call to store.add(), any # exceptions that occur there handle this same issue internally, # Since this is store-agnostic, should apply to all stores. initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPPreconditionFailed(explanation=msg, request=req, content_type='text/plain') except exception.Duplicate as e: msg = u"Attempt to upload duplicate image: %s" % e LOG.debug(msg) # NOTE(dosaboy): do not delete the image since it is likely that this # conflict is a result of another concurrent upload that will be # successful. notifier.error('image.upload', msg) raise webob.exc.HTTPConflict(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = u"Forbidden upload attempt: %s" % e LOG.debug(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except exception.StorageFull as e: msg = _("Image storage media is full: %s") % utils.exception_to_str(e) LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageWriteDenied as e: msg = (_("Insufficient permissions on image storage media: %s") % utils.exception_to_str(e)) LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = (_("Denying attempt to upload image larger than %d bytes.") % CONF.image_size_cap) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageQuotaFull as e: msg = (_("Denying attempt to upload image because it exceeds the ." "quota: %s") % utils.exception_to_str(e)) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError: #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. msg = _("Received HTTP error while uploading image %s") % image_id notifier.error('image.upload', msg) with excutils.save_and_reraise_exception(): LOG.exception(msg) safe_kill(req, image_id) except (ValueError, IOError) as e: msg = "Client disconnected before sending all data to backend" LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) except Exception as e: msg = _("Failed to upload image %s") % image_id LOG.exception(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPInternalServerError(explanation=msg, request=req, content_type='text/plain') return image_meta, location_data
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] db_api = glance.db.get_api(v1_mode=True) image_size = image_meta.get('size') try: # By default image_data will be passed as CooperativeReader object. # But if 'user_storage_quota' is enabled and 'remaining' is not None # then it will be passed as object of LimitingReader to # 'store_add_to_backend' method. image_data = utils.CooperativeReader(image_data) remaining = glance.api.common.check_quota( req.context, image_size, db_api, image_id=image_id) if remaining is not None: image_data = utils.LimitingReader(image_data, remaining) (uri, size, checksum, location_metadata) = store_api.store_add_to_backend( image_meta['id'], image_data, image_meta['size'], store, context=req.context) location_data = {'url': uri, 'metadata': location_metadata, 'status': 'active'} try: # recheck the quota in case there were simultaneous uploads that # did not provide the size glance.api.common.check_quota( req.context, size, db_api, image_id=image_id) except exception.StorageQuotaFull: with excutils.save_and_reraise_exception(): LOG.info(_LI('Cleaning up %s after exceeding ' 'the quota'), image_id) store_utils.safe_delete_from_backend( req.context, image_meta['id'], location_data) def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = (_("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % {'attr': attr, 'supplied': supplied, 'actual': actual}) LOG.error(msg) safe_kill(req, image_id, 'saving') initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d", {'image_id': image_id, 'checksum': checksum, 'size': size}) update_data = {'checksum': checksum, 'size': size} try: try: state = 'saving' image_meta = registry.update_image_metadata(req.context, image_id, update_data, from_state=state) except exception.Duplicate: image = registry.get_image_metadata(req.context, image_id) if image['status'] == 'deleted': raise exception.ImageNotFound() else: raise except exception.NotAuthenticated as e: # Delete image data due to possible token expiration. LOG.debug("Authentication error - the token may have " "expired during file upload. Deleting image data for " " %s " % image_id) initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPUnauthorized(explanation=e.msg, request=req) except exception.ImageNotFound: msg = _("Image %s could not be found after upload. The image may" " have been deleted during the upload.") % image_id LOG.info(msg) # NOTE(jculp): we need to clean up the datastore if an image # resource is deleted while the image data is being uploaded # # We get "location_data" from above call to store.add(), any # exceptions that occur there handle this same issue internally, # Since this is store-agnostic, should apply to all stores. initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPPreconditionFailed(explanation=msg, request=req, content_type='text/plain') except store_api.StoreAddDisabled: msg = _("Error in store configuration. Adding images to store " "is disabled.") LOG.exception(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPGone(explanation=msg, request=req, content_type='text/plain') except (store_api.Duplicate, exception.Duplicate) as e: msg = (_("Attempt to upload duplicate image: %s") % encodeutils.exception_to_unicode(e)) LOG.warn(msg) # NOTE(dosaboy): do not delete the image since it is likely that this # conflict is a result of another concurrent upload that will be # successful. notifier.error('image.upload', msg) raise webob.exc.HTTPConflict(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = (_("Forbidden upload attempt: %s") % encodeutils.exception_to_unicode(e)) LOG.warn(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except store_api.StorageFull as e: msg = (_("Image storage media is full: %s") % encodeutils.exception_to_unicode(e)) LOG.error(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except store_api.StorageWriteDenied as e: msg = (_("Insufficient permissions on image storage media: %s") % encodeutils.exception_to_unicode(e)) LOG.error(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = (_("Denying attempt to upload image larger than %d bytes.") % CONF.image_size_cap) LOG.warn(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageQuotaFull as e: msg = (_("Denying attempt to upload image because it exceeds the " "quota: %s") % encodeutils.exception_to_unicode(e)) LOG.warn(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError: # NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. msg = _LE("Received HTTP error while uploading image %s") % image_id notifier.error('image.upload', msg) with excutils.save_and_reraise_exception(): LOG.exception(msg) safe_kill(req, image_id, 'saving') except (ValueError, IOError) as e: msg = _("Client disconnected before sending all data to backend") LOG.warn(msg) safe_kill(req, image_id, 'saving') raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) except Exception as e: msg = _("Failed to upload image %s") % image_id LOG.exception(msg) safe_kill(req, image_id, 'saving') notifier.error('image.upload', msg) raise webob.exc.HTTPInternalServerError(explanation=msg, request=req, content_type='text/plain') return image_meta, location_data