def set_data(self, data, size=None): payload = format_image_notification(self.image) self.notifier.info('image.prepare', payload) try: self.image.set_data(data, size) except exception.StorageFull as e: msg = (_("Image storage media is full: %s") % utils.exception_to_str(e)) self.notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg) except exception.StorageWriteDenied as e: msg = (_("Insufficient permissions on image storage media: %s") % utils.exception_to_str(e)) self.notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg) except ValueError as e: msg = (_("Cannot save data for image %(image_id)s: %(error)s") % {'image_id': self.image.image_id, 'error': utils.exception_to_str(e)}) self.notifier.error('image.upload', msg) raise webob.exc.HTTPBadRequest(explanation= utils.exception_to_str(e)) except exception.Duplicate as e: msg = (_("Unable to upload duplicate image data for image" "%(image_id)s: %(error)s") % {'image_id': self.image.image_id, 'error': utils.exception_to_str(e)}) self.notifier.error('image.upload', msg) raise webob.exc.HTTPConflict(explanation=msg) except exception.Forbidden as e: msg = (_("Not allowed to upload image data for image %(image_id)s:" " %(error)s") % {'image_id': self.image.image_id, 'error': utils.exception_to_str(e)}) self.notifier.error('image.upload', msg) raise webob.exc.HTTPForbidden(explanation=msg) except exception.NotFound as e: msg = (_("Image %(image_id)s could not be found after upload." " The image may have been deleted during the upload:" " %(error)s") % {'image_id': self.image.image_id, 'error': utils.exception_to_str(e)}) self.notifier.error('image.upload', msg) raise webob.exc.HTTPNotFound(explanation=utils.exception_to_str(e)) except webob.exc.HTTPError as e: with excutils.save_and_reraise_exception(): msg = (_("Failed to upload image data for image %(image_id)s" " due to HTTP error: %(error)s") % {'image_id': self.image.image_id, 'error': utils.exception_to_str(e)}) self.notifier.error('image.upload', msg) except Exception as e: with excutils.save_and_reraise_exception(): msg = (_("Failed to upload image data for image %(image_id)s " "due to internal error: %(error)s") % {'image_id': self.image.image_id, 'error': utils.exception_to_str(e)}) self.notifier.error('image.upload', msg) else: payload = format_image_notification(self.image) self.notifier.info('image.upload', payload) self.notifier.info('image.activate', payload)
def size_checked_iter(response, image_meta, expected_size, image_iter, notifier): image_id = image_meta["id"] bytes_written = 0 def notify_image_sent_hook(env): image_send_notification(bytes_written, expected_size, image_meta, response.request, notifier) # Add hook to process after response is fully sent if "eventlet.posthooks" in response.request.environ: response.request.environ["eventlet.posthooks"].append((notify_image_sent_hook, (), {})) try: for chunk in image_iter: yield chunk bytes_written += len(chunk) except Exception as err: with excutils.save_and_reraise_exception(): msg = _("An error occurred reading from backend storage for " "image %(image_id)s: %(err)s") % { "image_id": image_id, "err": err, } LOG.error(msg) if expected_size != bytes_written: msg = _( "Backend storage for image %(image_id)s " "disconnected after writing only %(bytes_written)d " "bytes" ) % {"image_id": image_id, "bytes_written": bytes_written} LOG.error(msg) raise exception.GlanceException(_("Corrupt image download for " "image %(image_id)s") % {"image_id": image_id})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate( _("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) exceptions = { errno.EFBIG: exception.StorageFull(), errno.ENOSPC: exception.StorageFull(), errno.EACCES: exception.StorageWriteDenied() } raise exceptions.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), { 'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex }) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def cache_tee_iter(self, image_id, image_iter, image_checksum): try: current_checksum = hashlib.md5() with self.driver.open_for_write(image_id) as cache_file: for chunk in image_iter: try: cache_file.write(chunk) finally: current_checksum.update(chunk) yield chunk cache_file.flush() if (image_checksum and image_checksum != current_checksum.hexdigest()): msg = _("Checksum verification failed. Aborted " "caching of image '%s'.") % image_id raise exception.GlanceException(msg) except exception.GlanceException as e: with excutils.save_and_reraise_exception(): # image_iter has given us bad, (size_checked_iter has found a # bad length), or corrupt data (checksum is wrong). LOG.exception(utils.exception_to_str(e)) except Exception as e: LOG.exception(_LE("Exception encountered while tee'ing " "image '%(image_id)s' into cache: %(error)s. " "Continuing with response.") % {'image_id': image_id, 'error': utils.exception_to_str(e)}) # If no checksum provided continue responding even if # caching failed. for chunk in image_iter: yield chunk
def delete(self, location): """Takes a `glance.store.location.Location` object that indicates where to find the image file to delete :location `glance.store.location.Location` object, supplied from glance.store.location.get_location_from_uri() :raises NotFound if image does not exist """ file_path = '[%s] %s' % ( self.datastore_name, location.store_location.path[len(DS_URL_PREFIX):]) search_index_moref = self._service_content.searchIndex dc_moref = self._session.invoke_api(self._session.vim, 'FindByInventoryPath', search_index_moref, inventoryPath=self.datacenter_path) delete_task = self._session.invoke_api( self._session.vim, 'DeleteDatastoreFile_Task', self._service_content.fileManager, name=file_path, datacenter=dc_moref) try: self._session.wait_for_task(delete_task) except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_('Failed to delete image %(image)s content.') % {'image': location.image_id})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ loc = StoreLocation({"image_id": image_id}) if self.fs.exists(image_id): raise exception.Duplicate(_("GridFS already has an image at " "location %s") % loc.get_uri()) LOG.debug(_("Adding a new image to GridFS with id %s and size %s") % (image_id, image_size)) try: self.fs.put(image_file, _id=image_id) image = self._get_file(loc) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self.fs.delete(image_id) LOG.debug(_("Uploaded image %s, md5 %s, length %s to GridFS") % (image._id, image.md5, image.length)) return (loc.get_uri(), image.length, image.md5, {})
def show(self, req, id): """ Returns an iterator that can be used to retrieve an image's data along with the image metadata. :param req: The WSGI/Webob Request object :param id: The opaque image identifier :raises HTTPNotFound if image is not available to user """ self._enforce(req, "get_image") try: image_meta = self.get_active_image_meta_or_404(req, id) except HTTPNotFound: # provision for backward-compatibility breaking issue # catch the 404 exception and raise it after enforcing # the policy with excutils.save_and_reraise_exception(): self._enforce(req, "download_image") else: target = utils.create_mashup_dict(image_meta) self._enforce(req, "download_image", target=target) self._enforce_read_protected_props(image_meta, req) if image_meta.get("size") == 0: image_iterator = iter([]) else: image_iterator, size = self._get_from_store(req.context, image_meta["location"]) image_iterator = utils.cooperative_iter(image_iterator) image_meta["size"] = size or image_meta["size"] image_meta = redact_loc(image_meta) return {"image_iterator": image_iterator, "image_meta": image_meta}
def cache_tee_iter(self, image_id, image_iter, image_checksum): try: current_checksum = hashlib.md5() with self.driver.open_for_write(image_id) as cache_file: for chunk in image_iter: try: cache_file.write(chunk) finally: current_checksum.update(chunk) yield chunk cache_file.flush() if (image_checksum and image_checksum != current_checksum.hexdigest()): msg = _("Checksum verification failed. Aborted " "caching of image '%s'.") % image_id raise exception.GlanceException(msg) except exception.GlanceException as e: with excutils.save_and_reraise_exception(): # image_iter has given us bad, (size_checked_iter has found a # bad length), or corrupt data (checksum is wrong). LOG.exception(e) except Exception as e: LOG.exception(_("Exception encountered while tee'ing " "image '%(image_id)s' into cache: %(error)s. " "Continuing with response.") % {'image_id': image_id, 'error': e}) # If no checksum provided continue responding even if # caching failed. for chunk in image_iter: yield chunk
def _activate(self, req, image_id, location, location_metadata=None, from_state=None): """ Sets the image status to `active` and the image's location attribute. :param req: The WSGI/Webob Request object :param image_id: Opaque image identifier :param location: Location of where Glance stored this image :param location_metadata: a dictionary of storage specific information """ image_meta = {} image_meta["location"] = location image_meta["status"] = "active" if location_metadata: image_meta["location_data"] = [{"url": location, "metadata": location_metadata}] try: s = from_state image_meta_data = registry.update_image_metadata(req.context, image_id, image_meta, from_state=s) self.notifier.info("image.activate", redact_loc(image_meta_data)) self.notifier.info("image.update", redact_loc(image_meta_data)) return image_meta_data except exception.Duplicate: with excutils.save_and_reraise_exception(): # Delete image data since it has been supersceded by another # upload and re-raise. LOG.debug( "duplicate operation - deleting image data for " " %(id)s (location:%(location)s)" % {"id": image_id, "location": image_meta["location"]} ) upload_utils.initiate_deletion(req, image_meta["location"], image_id, CONF.delayed_delete) except exception.Invalid as e: msg = "Failed to activate image. Got error: %s" % utils.exception_to_str(e) LOG.debug(msg) raise HTTPBadRequest(explanation=msg, request=req, content_type="text/plain")
def delete(self, location): """Takes a `glance.store.location.Location` object that indicates where to find the image file to delete :location `glance.store.location.Location` object, supplied from glance.store.location.get_location_from_uri() :raises NotFound if image does not exist """ file_path = '[%s] %s' % (self.datastore_name, location.store_location. path[len(DS_URL_PREFIX):]) search_index_moref = self._service_content.searchIndex dc_moref = self._session.invoke_api(self._session.vim, 'FindByInventoryPath', search_index_moref, inventoryPath=self.datacenter_path) delete_task = self._session.invoke_api( self._session.vim, 'DeleteDatastoreFile_Task', self._service_content.fileManager, name=file_path, datacenter=dc_moref) try: self._session.wait_for_task(delete_task) except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to delete image %(image)s content.'), {'image': location.image_id})
def do_request(self, method, action, **kwargs): try: kwargs['headers'] = kwargs.get('headers', {}) kwargs['headers'].update(self.identity_headers or {}) res = super(RegistryClient, self).do_request(method, action, **kwargs) status = res.status request_id = res.getheader('x-openstack-request-id') msg = (_("Registry request %(method)s %(action)s HTTP %(status)s" " request id %(request_id)s") % { 'method': method, 'action': action, 'status': status, 'request_id': request_id }) LOG.debug(msg) except Exception as exc: with excutils.save_and_reraise_exception(): exc_name = exc.__class__.__name__ LOG.info( _("Registry client request %(method)s %(action)s " "raised %(exc_name)s"), { 'method': method, 'action': action, 'exc_name': exc_name }) return res
def rewind(self): try: self.data.seek(0) self._size = 0 self.checksum = hashlib.md5() except IOError: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to rewind image content'))
def open_for_write(self, image_id): """ Open a file for writing the image file for an image with supplied identifier. :param image_id: Image ID """ incomplete_path = self.get_image_filepath(image_id, 'incomplete') def set_attr(key, value): set_xattr(incomplete_path, key, value) def commit(): set_attr('hits', 0) final_path = self.get_image_filepath(image_id) LOG.debug("Fetch finished, moving " "'%(incomplete_path)s' to '%(final_path)s'", dict(incomplete_path=incomplete_path, final_path=final_path)) os.rename(incomplete_path, final_path) # Make sure that we "pop" the image from the queue... if self.is_queued(image_id): LOG.debug("Removing image '%s' from queue after " "caching it." % image_id) os.unlink(self.get_image_filepath(image_id, 'queue')) def rollback(e): set_attr('error', utils.exception_to_str(e)) invalid_path = self.get_image_filepath(image_id, 'invalid') LOG.debug("Fetch of cache file failed (%(e)s), rolling back by " "moving '%(incomplete_path)s' to " "'%(invalid_path)s'" % {'e': utils.exception_to_str(e), 'incomplete_path': incomplete_path, 'invalid_path': invalid_path}) os.rename(incomplete_path, invalid_path) try: with open(incomplete_path, 'wb') as cache_file: yield cache_file except Exception as e: with excutils.save_and_reraise_exception(): rollback(e) else: commit() finally: # if the generator filling the cache file neither raises an # exception, nor completes fetching all data, neither rollback # nor commit will have been called, so the incomplete file # will persist - in that case remove it as it is unusable # example: ^c from client fetch if os.path.exists(incomplete_path): rollback('incomplete fetch')
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate(_("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) exceptions = {errno.EFBIG: exception.StorageFull(), errno.ENOSPC: exception.StorageFull(), errno.EACCES: exception.StorageWriteDenied()} raise exceptions.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s", {'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex}) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed `glance.common.exception.UnexpectedStatus` if the upload request returned an unexpected status. The expected responses are 201 Created and 200 OK. """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation({ 'scheme': self.scheme, 'server_host': self.server_host, 'image_dir': self.store_image_dir, 'datacenter_path': self.datacenter_path, 'datastore_name': self.datastore_name, 'image_id': image_id }) cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = { 'Connection': 'Keep-Alive', 'Cookie': cookie, 'Transfer-Encoding': 'chunked' } try: conn = self._get_http_conn('PUT', loc, headers, content=image_file) res = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception( _('Failed to upload content of image ' '%(image)s') % {'image': image_id}) if res.status == httplib.CONFLICT: raise exception.Duplicate( _("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) if res.status not in (httplib.CREATED, httplib.OK): msg = (_('Failed to upload content of image %(image)s') % { 'image': image_id }) LOG.error(msg) raise exception.UnexpectedStatus(status=res.status, body=res.read()) return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def pop(self, i=-1): location = self.value.pop(i) try: self.image_proxy.store_utils.delete_image_location_from_backend( self.image_proxy.context, self.image_proxy.image.image_id, location) except Exception: with excutils.save_and_reraise_exception(): self.value.insert(i, location) return location
def add(self, image_id, image_file, image_size): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed `glance.common.exception.UnexpectedStatus` if the upload request returned an unexpected status. The expected responses are 201 Created and 200 OK. """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation({'scheme': self.scheme, 'server_host': self.server_host, 'image_dir': self.store_image_dir, 'datacenter_path': self.datacenter_path, 'datastore_name': self.datastore_name, 'image_id': image_id}) cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = {'Connection': 'Keep-Alive', 'Cookie': cookie, 'Transfer-Encoding': 'chunked'} try: conn = self._get_http_conn('PUT', loc, headers, content=image_file) res = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to upload content of image ' '%(image)s'), {'image': image_id}) if res.status == httplib.UNAUTHORIZED: self._create_session() image_file.rewind() raise exception.NotAuthenticated() if res.status == httplib.CONFLICT: raise exception.Duplicate(_("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) if res.status not in (httplib.CREATED, httplib.OK): msg = (_LE('Failed to upload content of image %(image)s') % {'image': image_id}) LOG.error(msg) raise exception.UnexpectedStatus(status=res.status, body=res.read()) return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def remove_path_on_error(path): """Protect code that wants to operate on PATH atomically. Any exception will cause PATH to be removed. :param path: File to work with """ try: yield except Exception: with excutils.save_and_reraise_exception(): delete_if_exists(path)
def pop(self, i=-1): location = self.value.pop(i) try: self.image_proxy.store_utils.delete_image_location_from_backend( self.image_proxy.context, self.image_proxy.image.image_id, location) except Exception: with excutils.save_and_reraise_exception(): self.value.insert(i, location) return location
def __init__(self, strategy=None): _driver = None _strategy = strategy if CONF.notifier_strategy != 'default': msg = _("notifier_strategy was deprecated in " "favor of `notification_driver`") LOG.warn(msg) # NOTE(flaper87): Use this to keep backwards # compatibility. We'll try to get an oslo.messaging # driver from the specified strategy. _strategy = strategy or CONF.notifier_strategy _driver = _STRATEGY_ALIASES.get(_strategy) publisher_id = CONF.default_publisher_id try: # NOTE(flaper87): Assume the user has configured # the transport url. self._transport = messaging.get_transport(CONF, aliases=_ALIASES) except messaging.DriverLoadFailure: # NOTE(flaper87): Catch driver load failures and re-raise # them *just* if the `transport_url` option was set. This # step is intended to keep backwards compatibility and avoid # weird behaviors (like exceptions on missing dependencies) # when the old notifier options are used. if CONF.transport_url is not None: with excutils.save_and_reraise_exception(): LOG.exception(_('Error loading the notifier')) # NOTE(flaper87): This needs to be checked # here because the `get_transport` call # registers `transport_url` into ConfigOpts. if not CONF.transport_url: # NOTE(flaper87): The next 3 lines help # with the migration to oslo.messaging. # Without them, gate tests won't know # what driver should be loaded. # Once this patch lands, devstack will be # updated and then these lines will be removed. url = None if _strategy in ['rabbit', 'qpid']: url = _strategy + '://' self._transport = messaging.get_transport(CONF, url, aliases=_ALIASES) self._notifier = messaging.Notifier(self._transport, driver=_driver, publisher_id=publisher_id)
def _query(self, location, method, depth=0): if depth > MAX_REDIRECTS: msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects.", { 'max_redirects': MAX_REDIRECTS }) LOG.debug(msg) raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) loc = location.store_location cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = {'Cookie': cookie} try: conn = self._get_http_conn(method, loc, headers) resp = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to access image %(image)s content.'), {'image': location.image_id}) if resp.status >= 400: if resp.status == httplib.UNAUTHORIZED: self._create_session() raise exception.NotAuthenticated() if resp.status == httplib.NOT_FOUND: msg = 'VMware datastore could not find image at URI.' LOG.debug(msg) raise exception.NotFound(msg) reason = (_('HTTP request returned a %(status)s status code.') % { 'status': resp.status }) LOG.info(reason) raise exception.BadStoreUri(message=reason) location_header = resp.getheader('location') if location_header: if resp.status not in (301, 302): reason = (_("The HTTP URL %(path)s attempted to redirect " "with an invalid %(status)s status code.") % { 'path': loc.path, 'status': resp.status }) LOG.info(reason) raise exception.BadStoreUri(message=reason) location_class = glance.store.location.Location new_loc = location_class(location.store_name, location.store_location.__class__, uri=location_header, image_id=location.image_id, store_specs=location.store_specs) return self._query(new_loc, method, depth + 1) content_length = int(resp.getheader('content-length', 0)) return (conn, resp, content_length)
def __init__(self, strategy=None): _driver = None _strategy = strategy if CONF.notifier_strategy != 'default': msg = _("notifier_strategy was deprecated in " "favor of `notification_driver`") LOG.warn(msg) # NOTE(flaper87): Use this to keep backwards # compatibility. We'll try to get an oslo.messaging # driver from the specified strategy. _strategy = strategy or CONF.notifier_strategy _driver = _STRATEGY_ALIASES.get(_strategy) publisher_id = CONF.default_publisher_id try: # NOTE(flaper87): Assume the user has configured # the transport url. self._transport = messaging.get_transport(CONF, aliases=_ALIASES) except messaging.DriverLoadFailure: # NOTE(flaper87): Catch driver load failures and re-raise # them *just* if the `transport_url` option was set. This # step is intended to keep backwards compatibility and avoid # weird behaviors (like exceptions on missing dependencies) # when the old notifier options are used. if CONF.transport_url is not None: with excutils.save_and_reraise_exception(): LOG.exception(_('Error loading the notifier')) # NOTE(flaper87): This needs to be checked # here because the `get_transport` call # registers `transport_url` into ConfigOpts. if not CONF.transport_url: # NOTE(flaper87): The next 3 lines help # with the migration to oslo.messaging. # Without them, gate tests won't know # what driver should be loaded. # Once this patch lands, devstack will be # updated and then these lines will be removed. url = None if _strategy in ['rabbit', 'qpid']: url = _strategy + '://' self._transport = messaging.get_transport(CONF, url, aliases=_ALIASES) self._notifier = messaging.Notifier(self._transport, driver=_driver, publisher_id=publisher_id)
def remove_path_on_error(path, remove=delete_if_exists): """Protect code that wants to operate on PATH atomically. Any exception will cause PATH to be removed. :param path: File to work with :param remove: Optional function to remove passed path """ try: yield except Exception: with excutils.save_and_reraise_exception(): remove(path)
def _activate(self, req, image_id, location, location_metadata=None, from_state=None): """ Sets the image status to `active` and the image's location attribute. :param req: The WSGI/Webob Request object :param image_id: Opaque image identifier :param location: Location of where Glance stored this image :param location_metadata: a dictionary of storage specific information """ image_meta = {} image_meta['location'] = location image_meta['status'] = 'active' if location_metadata: image_meta['location_data'] = [{ 'url': location, 'metadata': location_metadata }] try: s = from_state image_meta_data = registry.update_image_metadata(req.context, image_id, image_meta, from_state=s) self.notifier.info("image.activate", redact_loc(image_meta_data)) self.notifier.info("image.update", redact_loc(image_meta_data)) return image_meta_data except exception.Duplicate: with excutils.save_and_reraise_exception(): # Delete image data since it has been supersceded by another # upload and re-raise. LOG.debug( _("duplicate operation - deleting image data for " " %(id)s (location:%(location)s)") % { 'id': image_id, 'location': image_meta['location'] }) upload_utils.initiate_deletion(req, image_meta['location'], image_id, CONF.delayed_delete) except exception.Invalid as e: msg = _("Failed to activate image. Got error: %(e)s") % {'e': e} LOG.debug(msg) raise HTTPBadRequest(explanation=msg, request=req, content_type="text/plain")
def set_data(self, data, size=None): remaining = glance.api.common.check_quota(self.context, size, self.db_api, image_id=self.image.image_id) if remaining is not None: # NOTE(jbresnah) we are trying to enforce a quota, put a limit # reader on the data data = utils.LimitingReader(data, remaining) try: self.image.set_data(data, size=size) except exception.ImageSizeLimitExceeded: raise exception.StorageQuotaFull(image_size=size, remaining=remaining) # NOTE(jbresnah) If two uploads happen at the same time and neither # properly sets the size attribute[1] then there is a race condition # that will allow for the quota to be broken[2]. Thus we must recheck # the quota after the upload and thus after we know the size. # # Also, when an upload doesn't set the size properly then the call to # check_quota above returns None and so utils.LimitingReader is not # used above. Hence the store (e.g. filesystem store) may have to # download the entire file before knowing the actual file size. Here # also we need to check for the quota again after the image has been # downloaded to the store. # # [1] For e.g. when using chunked transfers the 'Content-Length' # header is not set. # [2] For e.g.: # - Upload 1 does not exceed quota but upload 2 exceeds quota. # Both uploads are to different locations # - Upload 2 completes before upload 1 and writes image.size. # - Immediately, upload 1 completes and (over)writes image.size # with the smaller size. # - Now, to glance, image has not exceeded quota but, in # reality, the quota has been exceeded. try: glance.api.common.check_quota(self.context, self.image.size, self.db_api, image_id=self.image.image_id) except exception.StorageQuotaFull: with excutils.save_and_reraise_exception(): LOG.info( _('Cleaning up %s after exceeding the quota.') % self.image.image_id) location = self.image.locations[0]['url'] glance.store.safe_delete_from_backend(self.context, location, self.image.image_id)
def new_task_executor(self, context): try: executor_cls = ('glance.async.%s_executor.' 'TaskExecutor' % CONF.task.task_executor) LOG.debug("Loading %s executor" % CONF.task.task_executor) executor = importutils.import_class(executor_cls) return executor(context, self.task_repo, self.image_repo, self.image_factory) except ImportError: with excutils.save_and_reraise_exception(): LOG.exception(_LE("Failed to load the %s executor provided " "in the config.") % CONF.task.task_executor)
def cooperative_iter(iter): """ Return an iterator which schedules after each iteration. This can prevent eventlet thread starvation. :param iter: an iterator to wrap """ try: for chunk in iter: sleep(0) yield chunk except Exception as err: with excutils.save_and_reraise_exception(): msg = _("Error: cooperative_iter exception %s") % err LOG.error(msg)
def cooperative_iter(iter): """ Return an iterator which schedules after each iteration. This can prevent eventlet thread starvation. :param iter: an iterator to wrap """ try: for chunk in iter: sleep(0) yield chunk except Exception as err: with excutils.save_and_reraise_exception(): msg = _("Error: cooperative_iter exception %s") % err LOG.error(msg)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d', image_name, order) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self._delete_image(loc.image, loc.snapshot) return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d', image_name, order) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self._delete_image(loc.image, loc.snapshot) return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def _query(self, location, method, depth=0): if depth > MAX_REDIRECTS: msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects.", {'max_redirects': MAX_REDIRECTS}) LOG.debug(msg) raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) loc = location.store_location cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = {'Cookie': cookie} try: conn = self._get_http_conn(method, loc, headers) resp = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to access image %(image)s content.'), {'image': location.image_id}) if resp.status >= 400: if resp.status == httplib.UNAUTHORIZED: self._create_session() raise exception.NotAuthenticated() if resp.status == httplib.NOT_FOUND: msg = 'VMware datastore could not find image at URI.' LOG.debug(msg) raise exception.NotFound(msg) reason = (_('HTTP request returned a %(status)s status code.') % {'status': resp.status}) LOG.info(reason) raise exception.BadStoreUri(message=reason) location_header = resp.getheader('location') if location_header: if resp.status not in (301, 302): reason = (_("The HTTP URL %(path)s attempted to redirect " "with an invalid %(status)s status code.") % {'path': loc.path, 'status': resp.status}) LOG.info(reason) raise exception.BadStoreUri(message=reason) location_class = glance.store.location.Location new_loc = location_class(location.store_name, location.store_location.__class__, uri=location_header, image_id=location.image_id, store_specs=location.store_specs) return self._query(new_loc, method, depth + 1) content_length = int(resp.getheader('content-length', 0)) return (conn, resp, content_length)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ loc = StoreLocation({'image_id': image_id}) if self.fs.exists(image_id): raise exception.Duplicate( _("GridFS already has an image at " "location %s") % loc.get_uri()) LOG.debug( _("Adding a new image to GridFS with id %(id)s and " "size %(size)s") % { 'id': image_id, 'size': image_size }) try: self.fs.put(image_file, _id=image_id) image = self._get_file(loc) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self.fs.delete(image_id) LOG.debug( _("Uploaded image %(id)s, md5 %(md5)s, length %(length)s " "to GridFS") % { 'id': image._id, 'md5': image.md5, 'length': image.length }) return (loc.get_uri(), image.length, image.md5, {})
def set_data(self, data, size=None): remaining = glance.api.common.check_quota( self.context, size, self.db_api, image_id=self.image.image_id) if remaining is not None: # NOTE(jbresnah) we are trying to enforce a quota, put a limit # reader on the data data = utils.LimitingReader(data, remaining) try: self.image.set_data(data, size=size) except exception.ImageSizeLimitExceeded: raise exception.StorageQuotaFull(image_size=size, remaining=remaining) # NOTE(jbresnah) If two uploads happen at the same time and neither # properly sets the size attribute[1] then there is a race condition # that will allow for the quota to be broken[2]. Thus we must recheck # the quota after the upload and thus after we know the size. # # Also, when an upload doesn't set the size properly then the call to # check_quota above returns None and so utils.LimitingReader is not # used above. Hence the store (e.g. filesystem store) may have to # download the entire file before knowing the actual file size. Here # also we need to check for the quota again after the image has been # downloaded to the store. # # [1] For e.g. when using chunked transfers the 'Content-Length' # header is not set. # [2] For e.g.: # - Upload 1 does not exceed quota but upload 2 exceeds quota. # Both uploads are to different locations # - Upload 2 completes before upload 1 and writes image.size. # - Immediately, upload 1 completes and (over)writes image.size # with the smaller size. # - Now, to glance, image has not exceeded quota but, in # reality, the quota has been exceeded. try: glance.api.common.check_quota( self.context, self.image.size, self.db_api, image_id=self.image.image_id) except exception.StorageQuotaFull: with excutils.save_and_reraise_exception(): LOG.info(_('Cleaning up %s after exceeding the quota.') % self.image.image_id) location = self.image.locations[0]['url'] glance.store.safe_delete_from_backend( self.context, location, self.image.image_id)
def set_image_data(image, uri, task_id): data_iter = None try: LOG.info(_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be " "imported") % {"data_uri": uri, "task_id": task_id}) data_iter = script_utils.get_image_data_iter(uri) image.set_data(data_iter) except Exception as e: with excutils.save_and_reraise_exception(): LOG.warn(_LW("Task %(task_id)s failed with exception %(error)s") % {"error": common_utils.exception_to_str(e), "task_id": task_id}) LOG.info(_LI("Task %(task_id)s: Could not import image file" " %(image_data)s") % {"image_data": uri, "task_id": task_id}) finally: if isinstance(data_iter, file): data_iter.close()
def set_image_data(image, uri, task_id): data_iter = None try: LOG.info(_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be " "imported") % {"data_uri": uri, "task_id": task_id}) data_iter = script_utils.get_image_data_iter(uri) image.set_data(data_iter) except Exception as e: with excutils.save_and_reraise_exception(): LOG.warn(_LW("Task %(task_id)s failed with exception %(error)s") % {"error": common_utils.exception_to_str(e), "task_id": task_id}) LOG.info(_LI("Task %(task_id)s: Could not import image file" " %(image_data)s") % {"image_data": uri, "task_id": task_id}) finally: if isinstance(data_iter, file): data_iter.close()
def setup_remote_pydev_debug(host, port): error_msg = _('Error setting up the debug environment. Verify that the' ' option pydev_worker_debug_host is pointing to a valid ' 'hostname or IP on which a pydev server is listening on' ' the port indicated by pydev_worker_debug_port.') try: try: from pydev import pydevd except ImportError: import pydevd pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True) return True except Exception: with excutils.save_and_reraise_exception(): LOG.exception(error_msg)
def setup_remote_pydev_debug(host, port): error_msg = _('Error setting up the debug environment. Verify that the' ' option pydev_worker_debug_host is pointing to a valid ' 'hostname or IP on which a pydev server is listening on' ' the port indicated by pydev_worker_debug_port.') try: try: from pydev import pydevd except ImportError: import pydevd pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True) return True except Exception: with excutils.save_and_reraise_exception(): LOG.exception(error_msg)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, and checksum :raises `glance.common.exception.Duplicate` if the image already existed """ image = SheepdogImage(self.addr, self.port, image_id, self.chunk_size) if image.exist(): raise exception.Duplicate(_("Sheepdog image %s already exists") % image_id) location = StoreLocation({'image': image_id}) checksum = hashlib.md5() image.create(image_size) try: total = left = image_size while left > 0: length = min(self.chunk_size, left) data = image_file.read(length) image.write(data, total - left, length) left -= length checksum.update(data) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): image.delete() return (location.get_uri(), image_size, checksum.hexdigest(), {})
def import_image(image_repo, image_factory, task_input, task_id, uri): original_image = create_image(image_repo, image_factory, task_input.get('image_properties'), task_id) # NOTE: set image status to saving just before setting data original_image.status = 'saving' image_repo.save(original_image) image_id = original_image.image_id # NOTE: Retrieving image from the database because the Image object # returned from create_image method does not have appropriate factories # wrapped around it. new_image = image_repo.get(image_id) set_image_data(new_image, uri, None) try: # NOTE: Check if the Image is not deleted after setting the data # before saving the active image. Here if image status is # saving, then new_image is saved as it contains updated location, # size, virtual_size and checksum information and the status of # new_image is already set to active in set_image_data() call. image = image_repo.get(image_id) if image.status == 'saving': image_repo.save(new_image) return image_id else: msg = _("The Image %(image_id)s object being created by this task " "%(task_id)s, is no longer in valid status for further " "processing.") % {"image_id": image_id, "task_id": task_id} raise exception.Conflict(msg) except (exception.Conflict, exception.NotFound, exception.NotAuthenticated): with excutils.save_and_reraise_exception(): if new_image.locations: for location in new_image.locations: store_utils.delete_image_location_from_backend( new_image.context, image_id, location)
def import_image(image_repo, image_factory, task_input, task_id, uri): original_image = create_image(image_repo, image_factory, task_input.get('image_properties'), task_id) # NOTE: set image status to saving just before setting data original_image.status = 'saving' image_repo.save(original_image) image_id = original_image.image_id # NOTE: Retrieving image from the database because the Image object # returned from create_image method does not have appropriate factories # wrapped around it. new_image = image_repo.get(image_id) set_image_data(new_image, uri, None) try: # NOTE: Check if the Image is not deleted after setting the data # before saving the active image. Here if image status is # saving, then new_image is saved as it contains updated location, # size, virtual_size and checksum information and the status of # new_image is already set to active in set_image_data() call. image = image_repo.get(image_id) if image.status == 'saving': image_repo.save(new_image) return image_id else: msg = _("The Image %(image_id)s object being created by this task " "%(task_id)s, is no longer in valid status for further " "processing.") % {"image_id": image_id, "task_id": task_id} raise exception.Conflict(msg) except (exception.Conflict, exception.NotFound): with excutils.save_and_reraise_exception(): if new_image.locations: for location in new_image.locations: store_utils.delete_image_location_from_backend( new_image.context, image_id, location)
def size_checked_iter(response, image_meta, expected_size, image_iter, notifier): image_id = image_meta['id'] bytes_written = 0 def notify_image_sent_hook(env): image_send_notification(bytes_written, expected_size, image_meta, response.request, notifier) # Add hook to process after response is fully sent if 'eventlet.posthooks' in response.request.environ: response.request.environ['eventlet.posthooks'].append( (notify_image_sent_hook, (), {})) try: for chunk in image_iter: yield chunk bytes_written += len(chunk) except Exception as err: with excutils.save_and_reraise_exception(): msg = (_("An error occurred reading from backend storage for " "image %(image_id)s: %(err)s") % { 'image_id': image_id, 'err': err }) LOG.error(msg) if expected_size != bytes_written: msg = (_("Backend storage for image %(image_id)s " "disconnected after writing only %(bytes_written)d " "bytes") % { 'image_id': image_id, 'bytes_written': bytes_written }) LOG.error(msg) raise exception.GlanceException( _("Corrupt image download for " "image %(image_id)s") % {'image_id': image_id})
def do_request(self, method, action, **kwargs): try: kwargs['headers'] = kwargs.get('headers', {}) kwargs['headers'].update(self.identity_headers or {}) res = super(RegistryClient, self).do_request(method, action, **kwargs) status = res.status request_id = res.getheader('x-openstack-request-id') msg = ("Registry request %(method)s %(action)s HTTP %(status)s" " request id %(request_id)s" % {'method': method, 'action': action, 'status': status, 'request_id': request_id}) LOG.debug(msg) except Exception as exc: with excutils.save_and_reraise_exception(): exc_name = exc.__class__.__name__ LOG.info(_("Registry client request %(method)s %(action)s " "raised %(exc_name)s"), {'method': method, 'action': action, 'exc_name': exc_name}) return res
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] try: (location, size, checksum, locations_metadata) = glance.store.store_add_to_backend( image_meta['id'], utils.CooperativeReader(image_data), image_meta['size'], store) def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = _("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % locals() LOG.error(msg) safe_kill(req, image_id) initiate_deletion(req, location, image_id, CONF.delayed_delete) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug( _("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), locals()) update_data = {'checksum': checksum, 'size': size} try: image_meta = registry.update_image_metadata( req.context, image_id, update_data) except exception.NotFound as e: msg = _("Image %s could not be found after upload. The image may " "have been deleted during the upload.") % image_id LOG.info(msg) # NOTE(jculp): we need to clean up the datastore if an image # resource is deleted while the image data is being uploaded # # We get "location" from above call to store.add(), any # exceptions that occur there handle this same issue internally, # Since this is store-agnostic, should apply to all stores. initiate_deletion(req, location, image_id, CONF.delayed_delete) raise webob.exc.HTTPPreconditionFailed(explanation=msg, request=req, content_type='text/plain') except exception.Duplicate as e: msg = _("Attempt to upload duplicate image: %s") % e LOG.debug(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPConflict(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = _("Forbidden upload attempt: %s") % e LOG.debug(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except exception.StorageFull as e: msg = _("Image storage media is full: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image storage media: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = (_("Denying attempt to upload image larger than %d bytes.") % CONF.image_size_cap) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError: #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. msg = _("Received HTTP error while uploading image %s") % image_id notifier.error('image.upload', msg) with excutils.save_and_reraise_exception(): LOG.exception(msg) safe_kill(req, image_id) except (ValueError, IOError) as e: msg = _("Client disconnected before sending all data to backend") LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) except Exception as e: msg = _("Failed to upload image %s") % image_id LOG.exception(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPInternalServerError(explanation=msg, request=req, content_type='text/plain') return image_meta, location, locations_metadata
def add(self, image_id, image_file, image_size, connection=None): location = self.create_location(image_id) if not connection: connection = self.get_connection(location) self._create_container_if_missing(location.container, connection) LOG.debug(_("Adding image object '%(obj_name)s' " "to Swift") % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str(int( math.ceil(float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug(_("Cannot determine image size. Adding as a " "segmented object to Swift.")) total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend with excutils.save_and_reraise_exception(): LOG.exception(_("Error during chunked upload to " "backend, deleting stale chunks")) self._delete_stale_chunks(connection, location.container, written_chunks) bytes_read = reader.bytes_read msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s") % {'chunk_name': chunk_name, 'chunk_id': chunk_id, 'total_chunks': total_chunks, 'bytes_read': bytes_read, 'chunk_etag': chunk_etag}) LOG.debug(msg) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if size == 0. LOG.debug(_("Deleting final zero-length chunk")) connection.delete_object(location.container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = {'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest} # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details return (location.get_uri(), image_size, obj_etag, {}) except swiftclient.ClientException as e: if e.http_status == httplib.CONFLICT: raise exception.Duplicate(_("Swift already has an image at " "this location")) msg = (_("Failed to add object to Swift.\n" "Got error from Swift: %(e)s") % {'e': e}) LOG.error(msg) raise glance.store.BackendException(msg)
def upload(self, req, image_id, data, size): image_repo = self.gateway.get_repo(req.context) image = None try: image = image_repo.get(image_id) image.status = 'saving' try: image_repo.save(image) image.set_data(data, size) image_repo.save(image) except exception.NotFound as e: msg = (_("Image %(id)s could not be found after upload." "The image may have been deleted during the upload: " "%(error)s Cleaning up the chunks uploaded") % { 'id': image_id, 'error': utils.exception_to_str(e) }) LOG.warn(msg) # NOTE(sridevi): Cleaning up the uploaded chunks. try: image.delete() except exception.NotFound: # NOTE(sridevi): Ignore this exception pass raise webob.exc.HTTPGone(explanation=msg, request=req, content_type='text/plain') except ValueError as e: LOG.debug("Cannot save data for image %(id)s: %(e)s", { 'id': image_id, 'e': utils.exception_to_str(e) }) self._restore(image_repo, image) raise webob.exc.HTTPBadRequest( explanation=utils.exception_to_str(e)) except exception.InvalidImageStatusTransition as e: msg = utils.exception_to_str(e) LOG.debug(msg) raise webob.exc.HTTPConflict(explanation=e.msg, request=req) except exception.Forbidden as e: msg = ("Not allowed to upload image data for image %s" % image_id) LOG.debug(msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req) except exception.NotFound as e: raise webob.exc.HTTPNotFound(explanation=e.msg) except exception.StorageFull as e: msg = _("Image storage media " "is full: %s") % utils.exception_to_str(e) LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) except exception.StorageQuotaFull as e: msg = _("Image exceeds the storage " "quota: %s") % utils.exception_to_str(e) LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) except exception.ImageSizeLimitExceeded as e: msg = _("The incoming image is " "too large: %s") % utils.exception_to_str(e) LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image " "storage media: %s") % utils.exception_to_str(e) LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req) except webob.exc.HTTPError as e: with excutils.save_and_reraise_exception(): LOG.error(_("Failed to upload image data due to HTTP error")) self._restore(image_repo, image) except Exception as e: with excutils.save_and_reraise_exception(): LOG.exception( _("Failed to upload image data due to " "internal error")) self._restore(image_repo, image)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d and size %d', image_name, order, image_size) if image_size == 0: LOG.warning(_("since image size is zero we will be doing " "resize-before-write for each chunk which " "will be considerably slower than normal")) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will # be slower so setting a higher chunk size may # speed things up a bit. if image_size == 0: length = offset + len(chunk) LOG.debug(_("resizing image to %s KiB") % (length / 1024)) image.resize(length) LOG.debug(_("writing chunk at offset %s") % (offset)) offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self._delete_image(loc.image, loc.snapshot) return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] db_api = glance.db.get_api() image_size = image_meta.get('size', None) try: remaining = glance.api.common.check_quota( req.context, image_size, db_api, image_id=image_id) if remaining is not None: image_data = utils.LimitingReader(image_data, remaining) (location, size, checksum, locations_metadata) = glance.store.store_add_to_backend( image_meta['id'], utils.CooperativeReader(image_data), image_meta['size'], store) try: # recheck the quota in case there were simultaneous uploads that # did not provide the size glance.api.common.check_quota( req.context, size, db_api, image_id=image_id) except exception.StorageQuotaFull: LOG.info(_('Cleaning up %s after exceeding the quota') % image_id) glance.store.safe_delete_from_backend( location, req.context, image_meta['id']) raise def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = (_("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % {'attr': attr, 'supplied': supplied, 'actual': actual}) LOG.error(msg) safe_kill(req, image_id) initiate_deletion(req, location, image_id, CONF.delayed_delete) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug(_("Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d"), {'image_id': image_id, 'checksum': checksum, 'size': size}) update_data = {'checksum': checksum, 'size': size} try: image_meta = registry.update_image_metadata(req.context, image_id, update_data) except exception.NotFound as e: msg = _("Image %s could not be found after upload. The image may " "have been deleted during the upload.") % image_id LOG.info(msg) # NOTE(jculp): we need to clean up the datastore if an image # resource is deleted while the image data is being uploaded # # We get "location" from above call to store.add(), any # exceptions that occur there handle this same issue internally, # Since this is store-agnostic, should apply to all stores. initiate_deletion(req, location, image_id, CONF.delayed_delete) raise webob.exc.HTTPPreconditionFailed(explanation=msg, request=req, content_type='text/plain') except exception.Duplicate as e: msg = _("Attempt to upload duplicate image: %s") % e LOG.debug(msg) # NOTE(dosaboy): do not delete the image since it is likely that this # conflict is a result of another concurrent upload that will be # successful. notifier.error('image.upload', msg) raise webob.exc.HTTPConflict(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = _("Forbidden upload attempt: %s") % e LOG.debug(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except exception.StorageFull as e: msg = _("Image storage media is full: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image storage media: %s") % e LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = (_("Denying attempt to upload image larger than %d bytes.") % CONF.image_size_cap) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageQuotaFull as e: msg = (_("Denying attempt to upload image because it exceeds the ." "quota: %s") % e) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError: #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. msg = _("Received HTTP error while uploading image %s") % image_id notifier.error('image.upload', msg) with excutils.save_and_reraise_exception(): LOG.exception(msg) safe_kill(req, image_id) except (ValueError, IOError) as e: msg = _("Client disconnected before sending all data to backend") LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) except Exception as e: msg = _("Failed to upload image %s") % image_id LOG.exception(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPInternalServerError(explanation=msg, request=req, content_type='text/plain') return image_meta, location, locations_metadata
def add(self, image_id, image_file, image_size, connection=None): location = self.create_location(image_id) if not connection: connection = self.get_connection(location) self._create_container_if_missing(location.container, connection) LOG.debug("Adding image object '%(obj_name)s' " "to Swift" % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str( int( math.ceil( float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug("Cannot determine image size. Adding as a " "segmented object to Swift.") total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend with excutils.save_and_reraise_exception(): LOG.exception( _("Error during chunked upload to " "backend, deleting stale chunks")) self._delete_stale_chunks(connection, location.container, written_chunks) bytes_read = reader.bytes_read msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s" % { 'chunk_name': chunk_name, 'chunk_id': chunk_id, 'total_chunks': total_chunks, 'bytes_read': bytes_read, 'chunk_etag': chunk_etag }) LOG.debug(msg) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if size == 0. LOG.debug("Deleting final zero-length chunk") connection.delete_object(location.container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = { 'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest } # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details if swift_store_utils.is_multiple_swift_store_accounts_enabled(): include_creds = False else: include_creds = True return (location.get_uri(credentials_included=include_creds), image_size, obj_etag, {}) except swiftclient.ClientException as e: if e.http_status == httplib.CONFLICT: raise exception.Duplicate( _("Swift already has an image at " "this location")) msg = (_("Failed to add object to Swift.\n" "Got error from Swift: %s") % utils.exception_to_str(e)) LOG.error(msg) raise glance.store.BackendException(msg)
def upload_data_to_store(req, image_meta, image_data, store, notifier): """ Upload image data to specified store. Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] db_api = glance.db.get_api() image_size = image_meta.get('size') try: remaining = glance.api.common.check_quota(req.context, image_size, db_api, image_id=image_id) if remaining is not None: image_data = utils.LimitingReader(image_data, remaining) (uri, size, checksum, location_metadata) = store_api.store_add_to_backend( image_meta['id'], utils.CooperativeReader(image_data), image_meta['size'], store) location_data = { 'url': uri, 'metadata': location_metadata, 'status': 'active' } try: # recheck the quota in case there were simultaneous uploads that # did not provide the size glance.api.common.check_quota(req.context, size, db_api, image_id=image_id) except exception.StorageQuotaFull: with excutils.save_and_reraise_exception(): LOG.info( _('Cleaning up %s after exceeding ' 'the quota') % image_id) store_utils.safe_delete_from_backend(req.context, image_meta['id'], location_data) def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: msg = (_("Supplied %(attr)s (%(supplied)s) and " "%(attr)s generated from uploaded image " "(%(actual)s) did not match. Setting image " "status to 'killed'.") % { 'attr': attr, 'supplied': supplied, 'actual': actual }) LOG.error(msg) safe_kill(req, image_id) initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) # Verify any supplied size/checksum value matches size/checksum # returned from store when adding image _kill_mismatched(image_meta, 'size', size) _kill_mismatched(image_meta, 'checksum', checksum) # Update the database with the checksum returned # from the backend store LOG.debug( "Updating image %(image_id)s data. " "Checksum set to %(checksum)s, size set " "to %(size)d", { 'image_id': image_id, 'checksum': checksum, 'size': size }) update_data = {'checksum': checksum, 'size': size} try: image_meta = registry.update_image_metadata( req.context, image_id, update_data) except exception.NotFound as e: msg = _("Image %s could not be found after upload. The image may " "have been deleted during the upload.") % image_id LOG.info(msg) # NOTE(jculp): we need to clean up the datastore if an image # resource is deleted while the image data is being uploaded # # We get "location_data" from above call to store.add(), any # exceptions that occur there handle this same issue internally, # Since this is store-agnostic, should apply to all stores. initiate_deletion(req, location_data, image_id) raise webob.exc.HTTPPreconditionFailed(explanation=msg, request=req, content_type='text/plain') except exception.Duplicate as e: msg = u"Attempt to upload duplicate image: %s" % e LOG.debug(msg) # NOTE(dosaboy): do not delete the image since it is likely that this # conflict is a result of another concurrent upload that will be # successful. notifier.error('image.upload', msg) raise webob.exc.HTTPConflict(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = u"Forbidden upload attempt: %s" % e LOG.debug(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req, content_type="text/plain") except exception.StorageFull as e: msg = _("Image storage media is full: %s") % utils.exception_to_str(e) LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageWriteDenied as e: msg = (_("Insufficient permissions on image storage media: %s") % utils.exception_to_str(e)) LOG.error(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req, content_type='text/plain') except exception.ImageSizeLimitExceeded as e: msg = (_("Denying attempt to upload image larger than %d bytes.") % CONF.image_size_cap) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except exception.StorageQuotaFull as e: msg = (_("Denying attempt to upload image because it exceeds the ." "quota: %s") % utils.exception_to_str(e)) LOG.info(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req, content_type='text/plain') except webob.exc.HTTPError: #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the # exception context and we must explicitly re-raise the # caught exception. msg = _("Received HTTP error while uploading image %s") % image_id notifier.error('image.upload', msg) with excutils.save_and_reraise_exception(): LOG.exception(msg) safe_kill(req, image_id) except (ValueError, IOError) as e: msg = "Client disconnected before sending all data to backend" LOG.debug(msg) safe_kill(req, image_id) raise webob.exc.HTTPBadRequest(explanation=msg, content_type="text/plain", request=req) except Exception as e: msg = _("Failed to upload image %s") % image_id LOG.exception(msg) safe_kill(req, image_id) notifier.error('image.upload', msg) raise webob.exc.HTTPInternalServerError(explanation=msg, request=req, content_type='text/plain') return image_meta, location_data
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %(name)s with order %(order)d and ' 'size %(size)d', {'name': text_type(image_name), 'order': order, 'size': image_size}) if image_size == 0: LOG.warning(_("since image size is zero we will be doing " "resize-before-write for each chunk which " "will be considerably slower than normal")) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: bytes_written = 0 offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will # be slower so setting a higher chunk size may # speed things up a bit. if image_size == 0: chunk_length = len(chunk) length = offset + chunk_length bytes_written += chunk_length LOG.debug("resizing image to %s KiB" % (length / units.Ki)) image.resize(length) LOG.debug("writing chunk at offset %s" % (offset)) offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except Exception: with excutils.save_and_reraise_exception(): # Delete image if one was created try: self._delete_image(loc.image, loc.snapshot) except exception.NotFound: pass # Make sure we send back the image size whether provided or inferred. if image_size == 0: image_size = bytes_written return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def upload(self, req, image_id, data, size): image_repo = self.gateway.get_repo(req.context) try: image = image_repo.get(image_id) image.status = 'saving' try: image_repo.save(image) image.set_data(data, size) image_repo.save(image) except exception.NotFound as e: msg = (_("Image %(id)s could not be found after upload." "The image may have been deleted during the upload: " "%(error)s Cleaning up the chunks uploaded") % {'id': image_id, 'error': e}) LOG.warn(msg) # NOTE(sridevi): Cleaning up the uploaded chunks. try: image.delete() except exception.NotFound: # NOTE(sridevi): Ignore this exception pass raise webob.exc.HTTPGone(explanation=msg, request=req, content_type='text/plain') except ValueError as e: LOG.debug("Cannot save data for image %(id)s: %(e)s", {'id': image_id, 'e': utils.exception_to_str(e)}) self._restore(image_repo, image) raise webob.exc.HTTPBadRequest(explanation= utils.exception_to_str(e)) except exception.InvalidImageStatusTransition as e: msg = utils.exception_to_str(e) LOG.debug(msg) raise webob.exc.HTTPConflict(explanation=e.msg, request=req) except exception.Forbidden as e: msg = ("Not allowed to upload image data for image %s" % image_id) LOG.debug(msg) raise webob.exc.HTTPForbidden(explanation=msg, request=req) except exception.NotFound as e: raise webob.exc.HTTPNotFound(explanation=e.msg) except exception.StorageFull as e: msg = _("Image storage media is full: %s") % e LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) except exception.StorageQuotaFull as e: msg = _("Image exceeds the storage quota: %s") % e LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) except exception.ImageSizeLimitExceeded as e: msg = _("The incoming image is too large: %s") % e LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image storage media: %s") % e LOG.error(msg) self._restore(image_repo, image) raise webob.exc.HTTPServiceUnavailable(explanation=msg, request=req) except webob.exc.HTTPError as e: with excutils.save_and_reraise_exception(): LOG.error(_("Failed to upload image data due to HTTP error")) self._restore(image_repo, image) except Exception as e: with excutils.save_and_reraise_exception(): LOG.exception(_("Failed to upload image data due to " "internal error")) self._restore(image_repo, image)
def open_for_write(self, image_id): """ Open a file for writing the image file for an image with supplied identifier. :param image_id: Image ID """ incomplete_path = self.get_image_filepath(image_id, 'incomplete') def commit(): with self.get_db() as db: final_path = self.get_image_filepath(image_id) LOG.debug( _("Fetch finished, moving " "'%(incomplete_path)s' to '%(final_path)s'"), dict(incomplete_path=incomplete_path, final_path=final_path)) os.rename(incomplete_path, final_path) # Make sure that we "pop" the image from the queue... if self.is_queued(image_id): os.unlink(self.get_image_filepath(image_id, 'queue')) filesize = os.path.getsize(final_path) now = time.time() db.execute( """INSERT INTO cached_images (image_id, last_accessed, last_modified, hits, size) VALUES (?, 0, ?, 0, ?)""", (image_id, now, filesize)) db.commit() def rollback(e): with self.get_db() as db: if os.path.exists(incomplete_path): invalid_path = self.get_image_filepath(image_id, 'invalid') LOG.debug( _("Fetch of cache file failed (%(e)s), rolling " "back by moving '%(incomplete_path)s' to " "'%(invalid_path)s'"), { 'e': e, 'incomplete_path': incomplete_path, 'invalid_path': invalid_path }) os.rename(incomplete_path, invalid_path) db.execute( """DELETE FROM cached_images WHERE image_id = ?""", (image_id, )) db.commit() try: with open(incomplete_path, 'wb') as cache_file: yield cache_file except Exception as e: with excutils.save_and_reraise_exception(): rollback(e) else: commit() finally: # if the generator filling the cache file neither raises an # exception, nor completes fetching all data, neither rollback # nor commit will have been called, so the incomplete file # will persist - in that case remove it as it is unusable # example: ^c from client fetch if os.path.exists(incomplete_path): rollback('incomplete fetch')
def rewind(self): try: self.data.seek(0) except IOError: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to rewind image content'))
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %(name)s with order %(order)d and ' 'size %(size)d', {'name': text_type(image_name), 'order': order, 'size': image_size}) if image_size == 0: LOG.warning(_("since image size is zero we will be doing " "resize-before-write for each chunk which " "will be considerably slower than normal")) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: bytes_written = 0 offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will # be slower so setting a higher chunk size may # speed things up a bit. if image_size == 0: chunk_length = len(chunk) length = offset + chunk_length bytes_written += chunk_length LOG.debug("resizing image to %s KiB" % (length / units.Ki)) image.resize(length) LOG.debug("writing chunk at offset %s" % (offset)) offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except Exception: with excutils.save_and_reraise_exception(): # Delete image if one was created try: self._delete_image(loc.image, loc.snapshot) except exception.NotFound: pass # Make sure we send back the image size whether provided or inferred. if image_size == 0: image_size = bytes_written return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def delete(self, req, id): """ Deletes the image and all its chunks from the Glance :param req: The WSGI/Webob Request object :param id: The opaque image identifier :raises HttpBadRequest if image registry is invalid :raises HttpNotFound if image or any chunk is not available :raises HttpUnauthorized if image or any chunk is not deleteable by the requesting user """ self._enforce(req, "delete_image") image = self.get_image_meta_or_404(req, id) if image["protected"]: msg = "Image is protected" LOG.debug(msg) raise HTTPForbidden(explanation=msg, request=req, content_type="text/plain") if image["status"] == "pending_delete": msg = "Forbidden to delete a %s image." % image["status"] LOG.debug(msg) raise HTTPForbidden(explanation=msg, request=req, content_type="text/plain") elif image["status"] == "deleted": msg = "Image %s not found." % id LOG.debug(msg) raise HTTPNotFound(explanation=msg, request=req, content_type="text/plain") if image["location"] and CONF.delayed_delete: status = "pending_delete" else: status = "deleted" ori_status = image["status"] try: # Update the image from the registry first, since we rely on it # for authorization checks. # See https://bugs.launchpad.net/glance/+bug/1065187 image = registry.update_image_metadata(req.context, id, {"status": status}) try: # The image's location field may be None in the case # of a saving or queued image, therefore don't ask a backend # to delete the image if the backend doesn't yet store it. # See https://bugs.launchpad.net/glance/+bug/747799 if image["location"]: upload_utils.initiate_deletion(req, image["location"], id, CONF.delayed_delete) except Exception: with excutils.save_and_reraise_exception(): registry.update_image_metadata(req.context, id, {"status": ori_status}) registry.delete_image_metadata(req.context, id) except exception.NotFound as e: msg = _("Failed to find image to delete: %s") % utils.exception_to_str(e) for line in msg.split("\n"): LOG.info(line) raise HTTPNotFound(explanation=msg, request=req, content_type="text/plain") except exception.Forbidden as e: msg = _("Forbidden to delete image: %s") % utils.exception_to_str(e) for line in msg.split("\n"): LOG.info(line) raise HTTPForbidden(explanation=msg, request=req, content_type="text/plain") else: self.notifier.info("image.delete", redact_loc(image)) return Response(body="", status=200)