def _test_upload_data_to_store_exception(self, exc_class, expected_class): req = unit_test_utils.get_fake_request() size = 10 image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" notifier = self.mox.CreateMockAnything() store = self.mox.CreateMockAnything() store.add( image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndRaise(exc_class) self.mox.StubOutWithMock(upload_utils, "safe_kill") upload_utils.safe_kill(req, image_meta['id'], 'saving') self.mox.ReplayAll() self.assertRaises(expected_class, upload_utils.upload_data_to_store, req, image_meta, image_data, store, notifier) self.mox.VerifyAll()
def test_upload_data_to_store(self): req = unit_test_utils.get_fake_request() location = "file://foo/bar" size = 10 checksum = "checksum" image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" notifier = self.mox.CreateMockAnything() store = self.mox.CreateMockAnything() store.add( image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndReturn((location, size, checksum, {})) self.mox.StubOutWithMock(registry, "update_image_metadata") update_data = {'checksum': checksum, 'size': size} registry.update_image_metadata(req.context, image_meta['id'], update_data ).AndReturn( image_meta.update(update_data)) self.mox.ReplayAll() actual_meta, actual_loc, loc_meta = upload_utils.upload_data_to_store( req, image_meta, image_data, store, notifier) self.mox.VerifyAll() self.assertEqual(actual_loc, location) self.assertEqual(actual_meta, image_meta.update(update_data))
def test_upload_data_to_store_not_found_after_upload(self): req = unit_test_utils.get_fake_request() location = "file://foo/bar" size = 10 checksum = "checksum" image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" notifier = self.mox.CreateMockAnything() store = self.mox.CreateMockAnything() store.add( image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndReturn((location, size, checksum)) self.mox.StubOutWithMock(registry, "update_image_metadata") update_data = {'checksum': checksum, 'size': size} registry.update_image_metadata(req.context, image_meta['id'], update_data ).AndRaise(exception.NotFound) self.mox.StubOutWithMock(upload_utils, "safe_kill") upload_utils.safe_kill(req, image_meta['id']) self.mox.ReplayAll() self.assertRaises(webob.exc.HTTPPreconditionFailed, upload_utils.upload_data_to_store, req, image_meta, image_data, store, notifier) self.mox.VerifyAll()
def _test_upload_data_to_store_exception_with_notify(self, exc_class, expected_class): req = unit_test_utils.get_fake_request() location = "file://foo/bar" size = 10 checksum = "checksum" image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" store = self.mox.CreateMockAnything() store.add( image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndRaise(exc_class) self.mox.StubOutWithMock(upload_utils, "safe_kill") upload_utils.safe_kill(req, image_meta['id']) notifier = self.mox.CreateMockAnything() notifier.error('image.upload', mox.IgnoreArg()) self.mox.ReplayAll() self.assertRaises(expected_class, upload_utils.upload_data_to_store, req, image_meta, image_data, store, notifier) self.mox.VerifyAll()
def test_upload_data_to_store_mismatch_checksum(self): req = unit_test_utils.get_fake_request() location = "file://foo/bar" size = 10 checksum = "checksum" image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" notifier = self.mox.CreateMockAnything() store = self.mox.CreateMockAnything() store.add( image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndReturn((location, size, checksum + "NOT", {})) self.mox.StubOutWithMock(registry, "update_image_metadata") update_data = {'checksum': checksum} registry.update_image_metadata(req.context, image_meta['id'], update_data).AndReturn( image_meta.update(update_data)) notifier.error('image.upload', mox.IgnoreArg()) self.mox.ReplayAll() self.assertRaises(webob.exc.HTTPBadRequest, upload_utils.upload_data_to_store, req, image_meta, image_data, store, notifier) self.mox.VerifyAll()
def test_upload_data_to_store_mismatch_checksum(self): req = unit_test_utils.get_fake_request() location = "file://foo/bar" size = 10 checksum = "checksum" image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" notifier = self.mox.CreateMockAnything() store = self.mox.CreateMockAnything() store.add( image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndReturn((location, size, checksum + "NOT")) self.mox.StubOutWithMock(registry, "update_image_metadata") update_data = {'checksum': checksum} registry.update_image_metadata(req.context, image_meta['id'], update_data).AndReturn( image_meta.update(update_data)) self.mox.ReplayAll() self.assertRaises(webob.exc.HTTPBadRequest, upload_utils.upload_data_to_store, req, image_meta, image_data, store, notifier) self.mox.VerifyAll()
def test_upload_data_to_store_not_found_after_upload(self): req = unit_test_utils.get_fake_request() location = "file://foo/bar" size = 10 checksum = "checksum" image_meta = {'id': unit_test_utils.UUID1, 'size': size} image_data = "blah" notifier = self.mox.CreateMockAnything() store = self.mox.CreateMockAnything() store.add(image_meta['id'], mox.IgnoreArg(), image_meta['size']).AndReturn((location, size, checksum, {})) self.mox.StubOutWithMock(registry, "update_image_metadata") update_data = {'checksum': checksum, 'size': size} registry.update_image_metadata(req.context, image_meta['id'], update_data).AndRaise( exception.NotFound) self.mox.StubOutWithMock(upload_utils, "initiate_deletion") upload_utils.initiate_deletion(req, location, image_meta['id'], mox.IsA(bool)) self.mox.StubOutWithMock(upload_utils, "safe_kill") upload_utils.safe_kill(req, image_meta['id']) notifier.error('image.upload', mox.IgnoreArg()) self.mox.ReplayAll() self.assertRaises(webob.exc.HTTPPreconditionFailed, upload_utils.upload_data_to_store, req, image_meta, image_data, store, notifier) self.mox.VerifyAll()
def test_bad_metadata_not_dict(self): store = self.mox.CreateMockAnything() store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, [])) store.__str__ = lambda: "hello" self.mox.ReplayAll() self.assertRaises(glance.store.BackendException, glance.store.store_add_to_backend, self.image_id, self.data, self.size, store) self.mox.VerifyAll()
def test_bad_metadata_not_dict(self): store = self.mox.CreateMockAnything() store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, [])) store.__str__().AndReturn(('hello')) self.mox.ReplayAll() self.assertRaises(glance.store.BackendException, glance.store.store_add_to_backend, self.image_id, self.data, self.size, store) self.mox.VerifyAll()
def _bad_metadata(self, in_metadata): store = self.mox.CreateMockAnything() store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, in_metadata)) store.__str__ = lambda: "hello" self.mox.ReplayAll() self.assertRaises(glance.store.BackendException, glance.store.store_add_to_backend, self.image_id, self.data, self.size, store) self.mox.VerifyAll()
def _bad_metadata(self, in_metadata): store = self.mox.CreateMockAnything() store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, in_metadata)) store.__str__().AndReturn(('hello')) self.mox.ReplayAll() self.assertRaises(glance.store.BackendException, glance.store.store_add_to_backend, self.image_id, self.data, self.size, store) self.mox.VerifyAll()
def _good_metadata(self, in_metadata): store = self.mox.CreateMockAnything() store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, in_metadata)) self.mox.ReplayAll() (location, size, checksum, metadata) = glance.store.store_add_to_backend(self.image_id, self.data, self.size, store) self.mox.VerifyAll() self.assertEqual(self.location, location) self.assertEqual(self.size, size) self.assertEqual(self.checksum, checksum) self.assertEqual(in_metadata, metadata)
def _upload(self, req, image_meta): """ Uploads the payload of the request to a backend store in Glance. If the `x-image-meta-store` header is set, Glance will attempt to use that store, if not, Glance will use the store set by the flag `default_store`. :param req: The WSGI/Webob Request object :param image_meta: Mapping of metadata about image :raises HTTPConflict if image already exists :retval The location where the image was stored """ copy_from = self._copy_from(req) if copy_from: image_data, image_size = self._get_from_store(copy_from) image_meta['size'] = image_size or image_meta['size'] else: try: req.get_content_type('application/octet-stream') except exception.InvalidContentType: self._safe_kill(req, image_meta['id']) msg = _("Content-Type must be application/octet-stream") logger.error(msg) raise HTTPBadRequest(explanation=msg) image_data = req.body_file if req.content_length: image_size = int(req.content_length) elif 'x-image-meta-size' in req.headers: image_size = int(req.headers['x-image-meta-size']) else: logger.debug(_("Got request with no content-length and no " "x-image-meta-size header")) image_size = 0 store_name = req.headers.get('x-image-meta-store', self.conf.default_store) store = self.get_store_or_400(req, store_name) image_id = image_meta['id'] logger.debug(_("Setting image %s to status 'saving'"), image_id) registry.update_image_metadata(req.context, image_id, {'status': 'saving'}) try: logger.debug(_("Uploading image data for image %(image_id)s " "to %(store_name)s store"), locals()) if image_size > IMAGE_SIZE_CAP: max_image_size = IMAGE_SIZE_CAP msg = _("Denying attempt to upload image larger than " "%(max_image_size)d. Supplied image size was " "%(image_size)d") % locals() logger.warn(msg) raise HTTPBadRequest(msg, request=req) location, size, checksum = store.add(image_meta['id'], image_data, image_size) # Verify any supplied checksum value matches checksum # returned from store when adding image supplied_checksum = image_meta.get('checksum') if supplied_checksum and supplied_checksum != checksum: msg = _("Supplied checksum (%(supplied_checksum)s) and " "checksum generated from uploaded image " "(%(checksum)s) did not match. Setting image " "status to 'killed'.") % locals() logger.error(msg) self._safe_kill(req, image_id) raise HTTPBadRequest(msg, content_type="text/plain", request=req) # Update the database with the checksum returned # from the backend store logger.debug(_("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), locals()) registry.update_image_metadata(req.context, image_id, {'checksum': checksum, 'size': size}) self.notifier.info('image.upload', image_meta) return location except exception.Duplicate, e: msg = _("Attempt to upload duplicate image: %s") % e logger.error(msg) self._safe_kill(req, image_id) self.notifier.error('image.upload', msg) raise HTTPConflict(msg, request=req)
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] try: location, size, checksum = store.add( image_meta['id'], utils.CooperativeReader(image_data), image_meta['size']) def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = _("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % locals() LOG.error(msg) safe_kill(req, image_id) initiate_deletion(req, location, image_id, CONF.delayed_delete) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug(_("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), locals()) update_data = {'checksum': checksum, 'size': size} try: image_meta = registry.update_image_metadata(req.context, image_id, update_data) except exception.NotFound as e: msg = _("Image %s could not be found after upload. The image may " "have been deleted during the upload.") % image_id LOG.info(msg) raise webob.exc.HTTPPreconditionFailed(explanation=msg, request=req, content_type='text/plain') except exception.Duplicate as e: msg = _("Attempt to upload duplicate image: %s") % e LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPConflict(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = _("Forbidden upload attempt: %s") % e LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except exception.StorageFull as e: msg = _("Image storage media is full: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image storage media: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = _("Denying attempt to upload image larger than %d bytes." % CONF.image_size_cap) LOG.info(msg) safe_kill(req, image_id) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError as e: LOG.exception(_("Received HTTP error while uploading image.")) safe_kill(req, image_id) #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. raise e except Exception as e: msg = _("Failed to upload image") LOG.exception(msg) safe_kill(req, image_id) raise webob.exc.HTTPInternalServerError(explanation=msg, request=req, content_type='text/plain') return image_meta, location
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] try: location, size, checksum = store.add( image_meta['id'], utils.CooperativeReader(image_data), image_meta['size']) def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = _("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % locals() LOG.error(msg) safe_kill(req, image_id) initiate_deletion(req, location, image_id, CONF.delayed_delete) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug( _("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), locals()) update_data = {'checksum': checksum, 'size': size} image_meta = registry.update_image_metadata(req.context, image_id, update_data) except exception.Duplicate as e: msg = _("Attempt to upload duplicate image: %s") % e LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPConflict(explanation=msg, request=req) except exception.Forbidden as e: msg = _("Forbidden upload attempt: %s") % e LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except exception.StorageFull as e: msg = _("Image storage media is full: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image storage media: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = _("Denying attempt to upload image larger than %d bytes." % CONF.image_size_cap) LOG.info(msg) safe_kill(req, image_id) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError as e: LOG.exception(_("Received HTTP error while uploading image.")) safe_kill(req, image_id) #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. raise e except Exception as e: LOG.exception(_("Failed to upload image")) safe_kill(req, image_id) raise webob.exc.HTTPInternalServerError(request=req) return image_meta, location
def _upload(self, req, image_meta): """ Uploads the payload of the request to a backend store in Glance. If the `x-image-meta-store` header is set, Glance will attempt to use that store, if not, Glance will use the store set by the flag `default_store`. :param req: The WSGI/Webob Request object :param image_meta: Mapping of metadata about image :raises HTTPConflict if image already exists :retval The location where the image was stored """ try: req.get_content_type('application/octet-stream') except exception.InvalidContentType: self._safe_kill(req, image_meta['id']) msg = "Content-Type must be application/octet-stream" logger.error(msg) raise HTTPBadRequest(explanation=msg) store_name = req.headers.get('x-image-meta-store', self.options['default_store']) store = self.get_store_or_400(req, store_name) image_id = image_meta['id'] logger.debug("Setting image %s to status 'saving'", image_id) registry.update_image_metadata(self.options, req.context, image_id, {'status': 'saving'}) try: logger.debug("Uploading image data for image %(image_id)s " "to %(store_name)s store", locals()) location, size, checksum = store.add(image_meta['id'], req.body_file) # Verify any supplied checksum value matches checksum # returned from store when adding image supplied_checksum = image_meta.get('checksum') if supplied_checksum and supplied_checksum != checksum: msg = ("Supplied checksum (%(supplied_checksum)s) and " "checksum generated from uploaded image " "(%(checksum)s) did not match. Setting image " "status to 'killed'.") % locals() logger.error(msg) self._safe_kill(req, image_id) raise HTTPBadRequest(msg, content_type="text/plain", request=req) # Update the database with the checksum returned # from the backend store logger.debug("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d", locals()) registry.update_image_metadata(self.options, req.context, image_id, {'checksum': checksum, 'size': size}) return location except exception.Duplicate, e: msg = ("Attempt to upload duplicate image: %s") % e logger.error(msg) self._safe_kill(req, image_id) raise HTTPConflict(msg, request=req)
def _upload(self, req, image_meta): """ Uploads the payload of the request to a backend store in Glance. If the `x-image-meta-store` header is set, Glance will attempt to use that store, if not, Glance will use the store set by the flag `default_store`. :param req: The WSGI/Webob Request object :param image_meta: Mapping of metadata about image :raises HTTPConflict if image already exists :retval The location where the image was stored """ try: req.get_content_type('application/octet-stream') except exception.InvalidContentType: self._safe_kill(req, image_meta['id']) msg = _("Content-Type must be application/octet-stream") logger.error(msg) raise HTTPBadRequest(explanation=msg) store_name = req.headers.get('x-image-meta-store', self.options['default_store']) store = self.get_store_or_400(req, store_name) image_id = image_meta['id'] logger.debug(_("Setting image %s to status 'saving'"), image_id) registry.update_image_metadata(self.options, req.context, image_id, {'status': 'saving'}) try: logger.debug( _("Uploading image data for image %(image_id)s " "to %(store_name)s store"), locals()) if req.content_length: image_size = int(req.content_length) elif 'x-image-meta-size' in req.headers: image_size = int(req.headers['x-image-meta-size']) else: logger.debug( _("Got request with no content-length and no " "x-image-meta-size header")) image_size = 0 location, size, checksum = store.add(image_meta['id'], req.body_file, image_size) # Verify any supplied checksum value matches checksum # returned from store when adding image supplied_checksum = image_meta.get('checksum') if supplied_checksum and supplied_checksum != checksum: msg = _("Supplied checksum (%(supplied_checksum)s) and " "checksum generated from uploaded image " "(%(checksum)s) did not match. Setting image " "status to 'killed'.") % locals() logger.error(msg) self._safe_kill(req, image_id) raise HTTPBadRequest(msg, content_type="text/plain", request=req) # Update the database with the checksum returned # from the backend store logger.debug( _("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), locals()) registry.update_image_metadata(self.options, req.context, image_id, { 'checksum': checksum, 'size': size }) self.notifier.info('image.upload', image_meta) return location except exception.Duplicate, e: msg = _("Attempt to upload duplicate image: %s") % e logger.error(msg) self._safe_kill(req, image_id) self.notifier.error('image.upload', msg) raise HTTPConflict(msg, request=req)
def _upload(self, req, image_meta): """ Uploads the payload of the request to a backend store in Glance. If the `x-image-meta-store` header is set, Glance will attempt to use that store, if not, Glance will use the store set by the flag `default_store`. :param req: The WSGI/Webob Request object :param image_meta: Mapping of metadata about image :raises HTTPConflict if image already exists :retval The location where the image was stored """ try: req.get_content_type("application/octet-stream") except exception.InvalidContentType: self._safe_kill(req, image_meta["id"]) msg = _("Content-Type must be application/octet-stream") logger.error(msg) raise HTTPBadRequest(explanation=msg) store_name = req.headers.get("x-image-meta-store", self.options["default_store"]) store = self.get_store_or_400(req, store_name) image_id = image_meta["id"] logger.debug(_("Setting image %s to status 'saving'"), image_id) registry.update_image_metadata(self.options, req.context, image_id, {"status": "saving"}) try: logger.debug(_("Uploading image data for image %(image_id)s " "to %(store_name)s store"), locals()) if req.content_length: image_size = int(req.content_length) elif "x-image-meta-size" in req.headers: image_size = int(req.headers["x-image-meta-size"]) else: logger.debug(_("Got request with no content-length and no " "x-image-meta-size header")) image_size = 0 location, size, checksum = store.add(image_meta["id"], req.body_file, image_size) # Verify any supplied checksum value matches checksum # returned from store when adding image supplied_checksum = image_meta.get("checksum") if supplied_checksum and supplied_checksum != checksum: msg = ( _( "Supplied checksum (%(supplied_checksum)s) and " "checksum generated from uploaded image " "(%(checksum)s) did not match. Setting image " "status to 'killed'." ) % locals() ) logger.error(msg) self._safe_kill(req, image_id) raise HTTPBadRequest(msg, content_type="text/plain", request=req) # Update the database with the checksum returned # from the backend store logger.debug( _("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), locals(), ) registry.update_image_metadata(self.options, req.context, image_id, {"checksum": checksum, "size": size}) self.notifier.info("image.upload", image_meta) return location except exception.Duplicate, e: msg = _("Attempt to upload duplicate image: %s") % e logger.error(msg) self._safe_kill(req, image_id) self.notifier.error("image.upload", msg) raise HTTPConflict(msg, request=req)