def set_data(self, data, size=None):
payload = format_image_notification(self.image)
self.notifier.info('image.prepare', payload)
try:
self.image.set_data(data, size)
except exception.StorageFull as e:
msg = (_("Image storage media is full: %s") %
utils.exception_to_str(e))
self.notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
except exception.StorageWriteDenied as e:
msg = (_("Insufficient permissions on image storage media: %s")
% utils.exception_to_str(e))
self.notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg)
except ValueError as e:
msg = (_("Cannot save data for image %(image_id)s: %(error)s") %
{'image_id': self.image.image_id,
'error': utils.exception_to_str(e)})
self.notifier.error('image.upload', msg)
raise webob.exc.HTTPBadRequest(explanation=
utils.exception_to_str(e))
except exception.Duplicate as e:
msg = (_("Unable to upload duplicate image data for image"
"%(image_id)s: %(error)s") %
{'image_id': self.image.image_id,
'error': utils.exception_to_str(e)})
self.notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg)
except exception.Forbidden as e:
msg = (_("Not allowed to upload image data for image %(image_id)s:"
" %(error)s") % {'image_id': self.image.image_id,
'error': utils.exception_to_str(e)})
self.notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg)
except exception.NotFound as e:
msg = (_("Image %(image_id)s could not be found after upload."
" The image may have been deleted during the upload:"
" %(error)s") % {'image_id': self.image.image_id,
'error': utils.exception_to_str(e)})
self.notifier.error('image.upload', msg)
raise webob.exc.HTTPNotFound(explanation=utils.exception_to_str(e))
except webob.exc.HTTPError as e:
with excutils.save_and_reraise_exception():
msg = (_("Failed to upload image data for image %(image_id)s"
" due to HTTP error: %(error)s") %
{'image_id': self.image.image_id,
'error': utils.exception_to_str(e)})
self.notifier.error('image.upload', msg)
except Exception as e:
with excutils.save_and_reraise_exception():
msg = (_("Failed to upload image data for image %(image_id)s "
"due to internal error: %(error)s") %
{'image_id': self.image.image_id,
'error': utils.exception_to_str(e)})
self.notifier.error('image.upload', msg)
else:
payload = format_image_notification(self.image)
self.notifier.info('image.upload', payload)
self.notifier.info('image.activate', payload)
def size_checked_iter(response, image_meta, expected_size, image_iter, notifier):
image_id = image_meta["id"]
bytes_written = 0
def notify_image_sent_hook(env):
image_send_notification(bytes_written, expected_size, image_meta, response.request, notifier)
# Add hook to process after response is fully sent
if "eventlet.posthooks" in response.request.environ:
response.request.environ["eventlet.posthooks"].append((notify_image_sent_hook, (), {}))
try:
for chunk in image_iter:
yield chunk
bytes_written += len(chunk)
except Exception as err:
with excutils.save_and_reraise_exception():
msg = _("An error occurred reading from backend storage for " "image %(image_id)s: %(err)s") % {
"image_id": image_id,
"err": err,
}
LOG.error(msg)
if expected_size != bytes_written:
msg = _(
"Backend storage for image %(image_id)s " "disconnected after writing only %(bytes_written)d " "bytes"
) % {"image_id": image_id, "bytes_written": bytes_written}
LOG.error(msg)
raise exception.GlanceException(_("Corrupt image download for " "image %(image_id)s") % {"image_id": image_id})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
datadir = self._find_best_datadir(image_size)
filepath = os.path.join(datadir, str(image_id))
if os.path.exists(filepath):
raise exception.Duplicate(
_("Image file %s already exists!") % filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)
exceptions = {
errno.EFBIG: exception.StorageFull(),
errno.ENOSPC: exception.StorageFull(),
errno.EACCES: exception.StorageWriteDenied()
}
raise exceptions.get(e.errno, e)
except Exception:
with excutils.save_and_reraise_exception():
self._delete_partial(filepath, image_id)
checksum_hex = checksum.hexdigest()
metadata = self._get_metadata()
LOG.debug(
_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s"), {
'bytes_written': bytes_written,
'filepath': filepath,
'checksum_hex': checksum_hex
})
return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def cache_tee_iter(self, image_id, image_iter, image_checksum):
try:
current_checksum = hashlib.md5()
with self.driver.open_for_write(image_id) as cache_file:
for chunk in image_iter:
try:
cache_file.write(chunk)
finally:
current_checksum.update(chunk)
yield chunk
cache_file.flush()
if (image_checksum and
image_checksum != current_checksum.hexdigest()):
msg = _("Checksum verification failed. Aborted "
"caching of image '%s'.") % image_id
raise exception.GlanceException(msg)
except exception.GlanceException as e:
with excutils.save_and_reraise_exception():
# image_iter has given us bad, (size_checked_iter has found a
# bad length), or corrupt data (checksum is wrong).
LOG.exception(utils.exception_to_str(e))
except Exception as e:
LOG.exception(_LE("Exception encountered while tee'ing "
"image '%(image_id)s' into cache: %(error)s. "
"Continuing with response.") %
{'image_id': image_id,
'error': utils.exception_to_str(e)})
# If no checksum provided continue responding even if
# caching failed.
for chunk in image_iter:
yield chunk
def delete(self, location):
"""Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = '[%s] %s' % (
self.datastore_name,
location.store_location.path[len(DS_URL_PREFIX):])
search_index_moref = self._service_content.searchIndex
dc_moref = self._session.invoke_api(self._session.vim,
'FindByInventoryPath',
search_index_moref,
inventoryPath=self.datacenter_path)
delete_task = self._session.invoke_api(
self._session.vim,
'DeleteDatastoreFile_Task',
self._service_content.fileManager,
name=file_path,
datacenter=dc_moref)
try:
self._session.wait_for_task(delete_task)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_('Failed to delete image %(image)s content.') %
{'image': location.image_id})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
loc = StoreLocation({"image_id": image_id})
if self.fs.exists(image_id):
raise exception.Duplicate(_("GridFS already has an image at " "location %s") % loc.get_uri())
LOG.debug(_("Adding a new image to GridFS with id %s and size %s") % (image_id, image_size))
try:
self.fs.put(image_file, _id=image_id)
image = self._get_file(loc)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self.fs.delete(image_id)
LOG.debug(_("Uploaded image %s, md5 %s, length %s to GridFS") % (image._id, image.md5, image.length))
return (loc.get_uri(), image.length, image.md5, {})
def show(self, req, id):
"""
Returns an iterator that can be used to retrieve an image's
data along with the image metadata.
:param req: The WSGI/Webob Request object
:param id: The opaque image identifier
:raises HTTPNotFound if image is not available to user
"""
self._enforce(req, "get_image")
try:
image_meta = self.get_active_image_meta_or_404(req, id)
except HTTPNotFound:
# provision for backward-compatibility breaking issue
# catch the 404 exception and raise it after enforcing
# the policy
with excutils.save_and_reraise_exception():
self._enforce(req, "download_image")
else:
target = utils.create_mashup_dict(image_meta)
self._enforce(req, "download_image", target=target)
self._enforce_read_protected_props(image_meta, req)
if image_meta.get("size") == 0:
image_iterator = iter([])
else:
image_iterator, size = self._get_from_store(req.context, image_meta["location"])
image_iterator = utils.cooperative_iter(image_iterator)
image_meta["size"] = size or image_meta["size"]
image_meta = redact_loc(image_meta)
return {"image_iterator": image_iterator, "image_meta": image_meta}
def cache_tee_iter(self, image_id, image_iter, image_checksum):
try:
current_checksum = hashlib.md5()
with self.driver.open_for_write(image_id) as cache_file:
for chunk in image_iter:
try:
cache_file.write(chunk)
finally:
current_checksum.update(chunk)
yield chunk
cache_file.flush()
if (image_checksum and
image_checksum != current_checksum.hexdigest()):
msg = _("Checksum verification failed. Aborted "
"caching of image '%s'.") % image_id
raise exception.GlanceException(msg)
except exception.GlanceException as e:
with excutils.save_and_reraise_exception():
# image_iter has given us bad, (size_checked_iter has found a
# bad length), or corrupt data (checksum is wrong).
LOG.exception(e)
except Exception as e:
LOG.exception(_("Exception encountered while tee'ing "
"image '%(image_id)s' into cache: %(error)s. "
"Continuing with response.") %
{'image_id': image_id,
'error': e})
# If no checksum provided continue responding even if
# caching failed.
for chunk in image_iter:
yield chunk
def _activate(self, req, image_id, location, location_metadata=None, from_state=None):
"""
Sets the image status to `active` and the image's location
attribute.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
:param location: Location of where Glance stored this image
:param location_metadata: a dictionary of storage specific information
"""
image_meta = {}
image_meta["location"] = location
image_meta["status"] = "active"
if location_metadata:
image_meta["location_data"] = [{"url": location, "metadata": location_metadata}]
try:
s = from_state
image_meta_data = registry.update_image_metadata(req.context, image_id, image_meta, from_state=s)
self.notifier.info("image.activate", redact_loc(image_meta_data))
self.notifier.info("image.update", redact_loc(image_meta_data))
return image_meta_data
except exception.Duplicate:
with excutils.save_and_reraise_exception():
# Delete image data since it has been supersceded by another
# upload and re-raise.
LOG.debug(
"duplicate operation - deleting image data for "
" %(id)s (location:%(location)s)" % {"id": image_id, "location": image_meta["location"]}
)
upload_utils.initiate_deletion(req, image_meta["location"], image_id, CONF.delayed_delete)
except exception.Invalid as e:
msg = "Failed to activate image. Got error: %s" % utils.exception_to_str(e)
LOG.debug(msg)
raise HTTPBadRequest(explanation=msg, request=req, content_type="text/plain")
def delete(self, location):
"""Takes a `glance.store.location.Location` object that indicates
where to find the image file to delete
:location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
file_path = '[%s] %s' % (self.datastore_name, location.store_location.
path[len(DS_URL_PREFIX):])
search_index_moref = self._service_content.searchIndex
dc_moref = self._session.invoke_api(self._session.vim,
'FindByInventoryPath',
search_index_moref,
inventoryPath=self.datacenter_path)
delete_task = self._session.invoke_api(
self._session.vim,
'DeleteDatastoreFile_Task',
self._service_content.fileManager,
name=file_path,
datacenter=dc_moref)
try:
self._session.wait_for_task(delete_task)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to delete image %(image)s content.'),
{'image': location.image_id})
def do_request(self, method, action, **kwargs):
try:
kwargs['headers'] = kwargs.get('headers', {})
kwargs['headers'].update(self.identity_headers or {})
res = super(RegistryClient,
self).do_request(method, action, **kwargs)
status = res.status
request_id = res.getheader('x-openstack-request-id')
msg = (_("Registry request %(method)s %(action)s HTTP %(status)s"
" request id %(request_id)s") % {
'method': method,
'action': action,
'status': status,
'request_id': request_id
})
LOG.debug(msg)
except Exception as exc:
with excutils.save_and_reraise_exception():
exc_name = exc.__class__.__name__
LOG.info(
_("Registry client request %(method)s %(action)s "
"raised %(exc_name)s"), {
'method': method,
'action': action,
'exc_name': exc_name
})
return res
def rewind(self):
try:
self.data.seek(0)
self._size = 0
self.checksum = hashlib.md5()
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
def open_for_write(self, image_id):
"""
Open a file for writing the image file for an image
with supplied identifier.
:param image_id: Image ID
"""
incomplete_path = self.get_image_filepath(image_id, 'incomplete')
def set_attr(key, value):
set_xattr(incomplete_path, key, value)
def commit():
set_attr('hits', 0)
final_path = self.get_image_filepath(image_id)
LOG.debug("Fetch finished, moving "
"'%(incomplete_path)s' to '%(final_path)s'",
dict(incomplete_path=incomplete_path,
final_path=final_path))
os.rename(incomplete_path, final_path)
# Make sure that we "pop" the image from the queue...
if self.is_queued(image_id):
LOG.debug("Removing image '%s' from queue after "
"caching it." % image_id)
os.unlink(self.get_image_filepath(image_id, 'queue'))
def rollback(e):
set_attr('error', utils.exception_to_str(e))
invalid_path = self.get_image_filepath(image_id, 'invalid')
LOG.debug("Fetch of cache file failed (%(e)s), rolling back by "
"moving '%(incomplete_path)s' to "
"'%(invalid_path)s'" %
{'e': utils.exception_to_str(e),
'incomplete_path': incomplete_path,
'invalid_path': invalid_path})
os.rename(incomplete_path, invalid_path)
try:
with open(incomplete_path, 'wb') as cache_file:
yield cache_file
except Exception as e:
with excutils.save_and_reraise_exception():
rollback(e)
else:
commit()
finally:
# if the generator filling the cache file neither raises an
# exception, nor completes fetching all data, neither rollback
# nor commit will have been called, so the incomplete file
# will persist - in that case remove it as it is unusable
# example: ^c from client fetch
if os.path.exists(incomplete_path):
rollback('incomplete fetch')
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
datadir = self._find_best_datadir(image_size)
filepath = os.path.join(datadir, str(image_id))
if os.path.exists(filepath):
raise exception.Duplicate(_("Image file %s already exists!")
% filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)
exceptions = {errno.EFBIG: exception.StorageFull(),
errno.ENOSPC: exception.StorageFull(),
errno.EACCES: exception.StorageWriteDenied()}
raise exceptions.get(e.errno, e)
except Exception:
with excutils.save_and_reraise_exception():
self._delete_partial(filepath, image_id)
checksum_hex = checksum.hexdigest()
metadata = self._get_metadata()
LOG.debug("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s",
{'bytes_written': bytes_written,
'filepath': filepath,
'checksum_hex': checksum_hex})
return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
`glance.common.exception.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({
'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id
})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {
'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'
}
try:
conn = self._get_http_conn('PUT', loc, headers, content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_('Failed to upload content of image '
'%(image)s') % {'image': image_id})
if res.status == httplib.CONFLICT:
raise exception.Duplicate(
_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_('Failed to upload content of image %(image)s') % {
'image': image_id
})
LOG.error(msg)
raise exception.UnexpectedStatus(status=res.status,
body=res.read())
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def pop(self, i=-1):
location = self.value.pop(i)
try:
self.image_proxy.store_utils.delete_image_location_from_backend(
self.image_proxy.context, self.image_proxy.image.image_id,
location)
except Exception:
with excutils.save_and_reraise_exception():
self.value.insert(i, location)
return location
def add(self, image_id, image_file, image_size):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
`glance.common.exception.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'}
try:
conn = self._get_http_conn('PUT', loc, headers,
content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to upload content of image '
'%(image)s'), {'image': image_id})
if res.status == httplib.UNAUTHORIZED:
self._create_session()
image_file.rewind()
raise exception.NotAuthenticated()
if res.status == httplib.CONFLICT:
raise exception.Duplicate(_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_LE('Failed to upload content of image %(image)s') %
{'image': image_id})
LOG.error(msg)
raise exception.UnexpectedStatus(status=res.status,
body=res.read())
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def remove_path_on_error(path):
"""Protect code that wants to operate on PATH atomically.
Any exception will cause PATH to be removed.
:param path: File to work with
"""
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
delete_if_exists(path)
def pop(self, i=-1):
location = self.value.pop(i)
try:
self.image_proxy.store_utils.delete_image_location_from_backend(
self.image_proxy.context,
self.image_proxy.image.image_id,
location)
except Exception:
with excutils.save_and_reraise_exception():
self.value.insert(i, location)
return location
def __init__(self, strategy=None):
_driver = None
_strategy = strategy
if CONF.notifier_strategy != 'default':
msg = _("notifier_strategy was deprecated in "
"favor of `notification_driver`")
LOG.warn(msg)
# NOTE(flaper87): Use this to keep backwards
# compatibility. We'll try to get an oslo.messaging
# driver from the specified strategy.
_strategy = strategy or CONF.notifier_strategy
_driver = _STRATEGY_ALIASES.get(_strategy)
publisher_id = CONF.default_publisher_id
try:
# NOTE(flaper87): Assume the user has configured
# the transport url.
self._transport = messaging.get_transport(CONF,
aliases=_ALIASES)
except messaging.DriverLoadFailure:
# NOTE(flaper87): Catch driver load failures and re-raise
# them *just* if the `transport_url` option was set. This
# step is intended to keep backwards compatibility and avoid
# weird behaviors (like exceptions on missing dependencies)
# when the old notifier options are used.
if CONF.transport_url is not None:
with excutils.save_and_reraise_exception():
LOG.exception(_('Error loading the notifier'))
# NOTE(flaper87): This needs to be checked
# here because the `get_transport` call
# registers `transport_url` into ConfigOpts.
if not CONF.transport_url:
# NOTE(flaper87): The next 3 lines help
# with the migration to oslo.messaging.
# Without them, gate tests won't know
# what driver should be loaded.
# Once this patch lands, devstack will be
# updated and then these lines will be removed.
url = None
if _strategy in ['rabbit', 'qpid']:
url = _strategy + '://'
self._transport = messaging.get_transport(CONF, url,
aliases=_ALIASES)
self._notifier = messaging.Notifier(self._transport,
driver=_driver,
publisher_id=publisher_id)
def _query(self, location, method, depth=0):
if depth > MAX_REDIRECTS:
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {
'max_redirects': MAX_REDIRECTS
})
LOG.debug(msg)
raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie}
try:
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to access image %(image)s content.'),
{'image': location.image_id})
if resp.status >= 400:
if resp.status == httplib.UNAUTHORIZED:
self._create_session()
raise exception.NotAuthenticated()
if resp.status == httplib.NOT_FOUND:
msg = 'VMware datastore could not find image at URI.'
LOG.debug(msg)
raise exception.NotFound(msg)
reason = (_('HTTP request returned a %(status)s status code.') % {
'status': resp.status
})
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.") % {
'path': loc.path,
'status': resp.status
})
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
location_class = glance.store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def __init__(self, strategy=None):
_driver = None
_strategy = strategy
if CONF.notifier_strategy != 'default':
msg = _("notifier_strategy was deprecated in "
"favor of `notification_driver`")
LOG.warn(msg)
# NOTE(flaper87): Use this to keep backwards
# compatibility. We'll try to get an oslo.messaging
# driver from the specified strategy.
_strategy = strategy or CONF.notifier_strategy
_driver = _STRATEGY_ALIASES.get(_strategy)
publisher_id = CONF.default_publisher_id
try:
# NOTE(flaper87): Assume the user has configured
# the transport url.
self._transport = messaging.get_transport(CONF, aliases=_ALIASES)
except messaging.DriverLoadFailure:
# NOTE(flaper87): Catch driver load failures and re-raise
# them *just* if the `transport_url` option was set. This
# step is intended to keep backwards compatibility and avoid
# weird behaviors (like exceptions on missing dependencies)
# when the old notifier options are used.
if CONF.transport_url is not None:
with excutils.save_and_reraise_exception():
LOG.exception(_('Error loading the notifier'))
# NOTE(flaper87): This needs to be checked
# here because the `get_transport` call
# registers `transport_url` into ConfigOpts.
if not CONF.transport_url:
# NOTE(flaper87): The next 3 lines help
# with the migration to oslo.messaging.
# Without them, gate tests won't know
# what driver should be loaded.
# Once this patch lands, devstack will be
# updated and then these lines will be removed.
url = None
if _strategy in ['rabbit', 'qpid']:
url = _strategy + '://'
self._transport = messaging.get_transport(CONF,
url,
aliases=_ALIASES)
self._notifier = messaging.Notifier(self._transport,
driver=_driver,
publisher_id=publisher_id)
def remove_path_on_error(path, remove=delete_if_exists):
"""Protect code that wants to operate on PATH atomically.
Any exception will cause PATH to be removed.
:param path: File to work with
:param remove: Optional function to remove passed path
"""
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
remove(path)
def _activate(self,
req,
image_id,
location,
location_metadata=None,
from_state=None):
"""
Sets the image status to `active` and the image's location
attribute.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
:param location: Location of where Glance stored this image
:param location_metadata: a dictionary of storage specific information
"""
image_meta = {}
image_meta['location'] = location
image_meta['status'] = 'active'
if location_metadata:
image_meta['location_data'] = [{
'url': location,
'metadata': location_metadata
}]
try:
s = from_state
image_meta_data = registry.update_image_metadata(req.context,
image_id,
image_meta,
from_state=s)
self.notifier.info("image.activate", redact_loc(image_meta_data))
self.notifier.info("image.update", redact_loc(image_meta_data))
return image_meta_data
except exception.Duplicate:
with excutils.save_and_reraise_exception():
# Delete image data since it has been supersceded by another
# upload and re-raise.
LOG.debug(
_("duplicate operation - deleting image data for "
" %(id)s (location:%(location)s)") % {
'id': image_id,
'location': image_meta['location']
})
upload_utils.initiate_deletion(req, image_meta['location'],
image_id, CONF.delayed_delete)
except exception.Invalid as e:
msg = _("Failed to activate image. Got error: %(e)s") % {'e': e}
LOG.debug(msg)
raise HTTPBadRequest(explanation=msg,
request=req,
content_type="text/plain")
def set_data(self, data, size=None):
remaining = glance.api.common.check_quota(self.context,
size,
self.db_api,
image_id=self.image.image_id)
if remaining is not None:
# NOTE(jbresnah) we are trying to enforce a quota, put a limit
# reader on the data
data = utils.LimitingReader(data, remaining)
try:
self.image.set_data(data, size=size)
except exception.ImageSizeLimitExceeded:
raise exception.StorageQuotaFull(image_size=size,
remaining=remaining)
# NOTE(jbresnah) If two uploads happen at the same time and neither
# properly sets the size attribute[1] then there is a race condition
# that will allow for the quota to be broken[2]. Thus we must recheck
# the quota after the upload and thus after we know the size.
#
# Also, when an upload doesn't set the size properly then the call to
# check_quota above returns None and so utils.LimitingReader is not
# used above. Hence the store (e.g. filesystem store) may have to
# download the entire file before knowing the actual file size. Here
# also we need to check for the quota again after the image has been
# downloaded to the store.
#
# [1] For e.g. when using chunked transfers the 'Content-Length'
# header is not set.
# [2] For e.g.:
# - Upload 1 does not exceed quota but upload 2 exceeds quota.
# Both uploads are to different locations
# - Upload 2 completes before upload 1 and writes image.size.
# - Immediately, upload 1 completes and (over)writes image.size
# with the smaller size.
# - Now, to glance, image has not exceeded quota but, in
# reality, the quota has been exceeded.
try:
glance.api.common.check_quota(self.context,
self.image.size,
self.db_api,
image_id=self.image.image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(
_('Cleaning up %s after exceeding the quota.') %
self.image.image_id)
location = self.image.locations[0]['url']
glance.store.safe_delete_from_backend(self.context, location,
self.image.image_id)
def new_task_executor(self, context):
try:
executor_cls = ('glance.async.%s_executor.'
'TaskExecutor' % CONF.task.task_executor)
LOG.debug("Loading %s executor" % CONF.task.task_executor)
executor = importutils.import_class(executor_cls)
return executor(context,
self.task_repo,
self.image_repo,
self.image_factory)
except ImportError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to load the %s executor provided "
"in the config.") % CONF.task.task_executor)
def cooperative_iter(iter):
"""
Return an iterator which schedules after each
iteration. This can prevent eventlet thread starvation.
:param iter: an iterator to wrap
"""
try:
for chunk in iter:
sleep(0)
yield chunk
except Exception as err:
with excutils.save_and_reraise_exception():
msg = _("Error: cooperative_iter exception %s") % err
LOG.error(msg)
def cooperative_iter(iter):
"""
Return an iterator which schedules after each
iteration. This can prevent eventlet thread starvation.
:param iter: an iterator to wrap
"""
try:
for chunk in iter:
sleep(0)
yield chunk
except Exception as err:
with excutils.save_and_reraise_exception():
msg = _("Error: cooperative_iter exception %s") % err
LOG.error(msg)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d', image_name, order)
try:
loc = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
try:
with rbd.Image(ioctx, image_name) as image:
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
for chunk in chunks:
offset += image.write(chunk, offset)
checksum.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self._delete_image(loc.image, loc.snapshot)
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d', image_name, order)
try:
loc = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
try:
with rbd.Image(ioctx, image_name) as image:
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
for chunk in chunks:
offset += image.write(chunk, offset)
checksum.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self._delete_image(loc.image, loc.snapshot)
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def _query(self, location, method, depth=0):
if depth > MAX_REDIRECTS:
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
"redirects.", {'max_redirects': MAX_REDIRECTS})
LOG.debug(msg)
raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
loc = location.store_location
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie}
try:
conn = self._get_http_conn(method, loc, headers)
resp = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to access image %(image)s content.'),
{'image': location.image_id})
if resp.status >= 400:
if resp.status == httplib.UNAUTHORIZED:
self._create_session()
raise exception.NotAuthenticated()
if resp.status == httplib.NOT_FOUND:
msg = 'VMware datastore could not find image at URI.'
LOG.debug(msg)
raise exception.NotFound(msg)
reason = (_('HTTP request returned a %(status)s status code.')
% {'status': resp.status})
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
location_header = resp.getheader('location')
if location_header:
if resp.status not in (301, 302):
reason = (_("The HTTP URL %(path)s attempted to redirect "
"with an invalid %(status)s status code.")
% {'path': loc.path, 'status': resp.status})
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
location_class = glance.store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)
return self._query(new_loc, method, depth + 1)
content_length = int(resp.getheader('content-length', 0))
return (conn, resp, content_length)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
loc = StoreLocation({'image_id': image_id})
if self.fs.exists(image_id):
raise exception.Duplicate(
_("GridFS already has an image at "
"location %s") % loc.get_uri())
LOG.debug(
_("Adding a new image to GridFS with id %(id)s and "
"size %(size)s") % {
'id': image_id,
'size': image_size
})
try:
self.fs.put(image_file, _id=image_id)
image = self._get_file(loc)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self.fs.delete(image_id)
LOG.debug(
_("Uploaded image %(id)s, md5 %(md5)s, length %(length)s "
"to GridFS") % {
'id': image._id,
'md5': image.md5,
'length': image.length
})
return (loc.get_uri(), image.length, image.md5, {})
def set_data(self, data, size=None):
remaining = glance.api.common.check_quota(
self.context, size, self.db_api, image_id=self.image.image_id)
if remaining is not None:
# NOTE(jbresnah) we are trying to enforce a quota, put a limit
# reader on the data
data = utils.LimitingReader(data, remaining)
try:
self.image.set_data(data, size=size)
except exception.ImageSizeLimitExceeded:
raise exception.StorageQuotaFull(image_size=size,
remaining=remaining)
# NOTE(jbresnah) If two uploads happen at the same time and neither
# properly sets the size attribute[1] then there is a race condition
# that will allow for the quota to be broken[2]. Thus we must recheck
# the quota after the upload and thus after we know the size.
#
# Also, when an upload doesn't set the size properly then the call to
# check_quota above returns None and so utils.LimitingReader is not
# used above. Hence the store (e.g. filesystem store) may have to
# download the entire file before knowing the actual file size. Here
# also we need to check for the quota again after the image has been
# downloaded to the store.
#
# [1] For e.g. when using chunked transfers the 'Content-Length'
# header is not set.
# [2] For e.g.:
# - Upload 1 does not exceed quota but upload 2 exceeds quota.
# Both uploads are to different locations
# - Upload 2 completes before upload 1 and writes image.size.
# - Immediately, upload 1 completes and (over)writes image.size
# with the smaller size.
# - Now, to glance, image has not exceeded quota but, in
# reality, the quota has been exceeded.
try:
glance.api.common.check_quota(
self.context, self.image.size, self.db_api,
image_id=self.image.image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(_('Cleaning up %s after exceeding the quota.')
% self.image.image_id)
location = self.image.locations[0]['url']
glance.store.safe_delete_from_backend(
self.context, location, self.image.image_id)
def set_image_data(image, uri, task_id):
data_iter = None
try:
LOG.info(_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be "
"imported") % {"data_uri": uri, "task_id": task_id})
data_iter = script_utils.get_image_data_iter(uri)
image.set_data(data_iter)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.warn(_LW("Task %(task_id)s failed with exception %(error)s") %
{"error": common_utils.exception_to_str(e),
"task_id": task_id})
LOG.info(_LI("Task %(task_id)s: Could not import image file"
" %(image_data)s") % {"image_data": uri,
"task_id": task_id})
finally:
if isinstance(data_iter, file):
data_iter.close()
def set_image_data(image, uri, task_id):
data_iter = None
try:
LOG.info(_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be "
"imported") % {"data_uri": uri, "task_id": task_id})
data_iter = script_utils.get_image_data_iter(uri)
image.set_data(data_iter)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.warn(_LW("Task %(task_id)s failed with exception %(error)s") %
{"error": common_utils.exception_to_str(e),
"task_id": task_id})
LOG.info(_LI("Task %(task_id)s: Could not import image file"
" %(image_data)s") % {"image_data": uri,
"task_id": task_id})
finally:
if isinstance(data_iter, file):
data_iter.close()
def setup_remote_pydev_debug(host, port):
error_msg = _('Error setting up the debug environment. Verify that the'
' option pydev_worker_debug_host is pointing to a valid '
'hostname or IP on which a pydev server is listening on'
' the port indicated by pydev_worker_debug_port.')
try:
try:
from pydev import pydevd
except ImportError:
import pydevd
pydevd.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
return True
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(error_msg)
def setup_remote_pydev_debug(host, port):
error_msg = _('Error setting up the debug environment. Verify that the'
' option pydev_worker_debug_host is pointing to a valid '
'hostname or IP on which a pydev server is listening on'
' the port indicated by pydev_worker_debug_port.')
try:
try:
from pydev import pydevd
except ImportError:
import pydevd
pydevd.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
return True
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(error_msg)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.chunk_size)
if image.exist():
raise exception.Duplicate(_("Sheepdog image %s already exists")
% image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
try:
total = left = image_size
while left > 0:
length = min(self.chunk_size, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
image.delete()
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def import_image(image_repo, image_factory, task_input, task_id, uri):
original_image = create_image(image_repo, image_factory,
task_input.get('image_properties'), task_id)
# NOTE: set image status to saving just before setting data
original_image.status = 'saving'
image_repo.save(original_image)
image_id = original_image.image_id
# NOTE: Retrieving image from the database because the Image object
# returned from create_image method does not have appropriate factories
# wrapped around it.
new_image = image_repo.get(image_id)
set_image_data(new_image, uri, None)
try:
# NOTE: Check if the Image is not deleted after setting the data
# before saving the active image. Here if image status is
# saving, then new_image is saved as it contains updated location,
# size, virtual_size and checksum information and the status of
# new_image is already set to active in set_image_data() call.
image = image_repo.get(image_id)
if image.status == 'saving':
image_repo.save(new_image)
return image_id
else:
msg = _("The Image %(image_id)s object being created by this task "
"%(task_id)s, is no longer in valid status for further "
"processing.") % {"image_id": image_id,
"task_id": task_id}
raise exception.Conflict(msg)
except (exception.Conflict, exception.NotFound,
exception.NotAuthenticated):
with excutils.save_and_reraise_exception():
if new_image.locations:
for location in new_image.locations:
store_utils.delete_image_location_from_backend(
new_image.context,
image_id,
location)
def import_image(image_repo, image_factory, task_input, task_id, uri):
original_image = create_image(image_repo, image_factory,
task_input.get('image_properties'), task_id)
# NOTE: set image status to saving just before setting data
original_image.status = 'saving'
image_repo.save(original_image)
image_id = original_image.image_id
# NOTE: Retrieving image from the database because the Image object
# returned from create_image method does not have appropriate factories
# wrapped around it.
new_image = image_repo.get(image_id)
set_image_data(new_image, uri, None)
try:
# NOTE: Check if the Image is not deleted after setting the data
# before saving the active image. Here if image status is
# saving, then new_image is saved as it contains updated location,
# size, virtual_size and checksum information and the status of
# new_image is already set to active in set_image_data() call.
image = image_repo.get(image_id)
if image.status == 'saving':
image_repo.save(new_image)
return image_id
else:
msg = _("The Image %(image_id)s object being created by this task "
"%(task_id)s, is no longer in valid status for further "
"processing.") % {"image_id": image_id,
"task_id": task_id}
raise exception.Conflict(msg)
except (exception.Conflict, exception.NotFound):
with excutils.save_and_reraise_exception():
if new_image.locations:
for location in new_image.locations:
store_utils.delete_image_location_from_backend(
new_image.context,
image_id,
location)
def size_checked_iter(response, image_meta, expected_size, image_iter,
notifier):
image_id = image_meta['id']
bytes_written = 0
def notify_image_sent_hook(env):
image_send_notification(bytes_written, expected_size, image_meta,
response.request, notifier)
# Add hook to process after response is fully sent
if 'eventlet.posthooks' in response.request.environ:
response.request.environ['eventlet.posthooks'].append(
(notify_image_sent_hook, (), {}))
try:
for chunk in image_iter:
yield chunk
bytes_written += len(chunk)
except Exception as err:
with excutils.save_and_reraise_exception():
msg = (_("An error occurred reading from backend storage for "
"image %(image_id)s: %(err)s") % {
'image_id': image_id,
'err': err
})
LOG.error(msg)
if expected_size != bytes_written:
msg = (_("Backend storage for image %(image_id)s "
"disconnected after writing only %(bytes_written)d "
"bytes") % {
'image_id': image_id,
'bytes_written': bytes_written
})
LOG.error(msg)
raise exception.GlanceException(
_("Corrupt image download for "
"image %(image_id)s") % {'image_id': image_id})
def do_request(self, method, action, **kwargs):
try:
kwargs['headers'] = kwargs.get('headers', {})
kwargs['headers'].update(self.identity_headers or {})
res = super(RegistryClient, self).do_request(method,
action,
**kwargs)
status = res.status
request_id = res.getheader('x-openstack-request-id')
msg = ("Registry request %(method)s %(action)s HTTP %(status)s"
" request id %(request_id)s" %
{'method': method, 'action': action,
'status': status, 'request_id': request_id})
LOG.debug(msg)
except Exception as exc:
with excutils.save_and_reraise_exception():
exc_name = exc.__class__.__name__
LOG.info(_("Registry client request %(method)s %(action)s "
"raised %(exc_name)s"),
{'method': method, 'action': action,
'exc_name': exc_name})
return res
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
try:
(location, size, checksum,
locations_metadata) = glance.store.store_add_to_backend(
image_meta['id'], utils.CooperativeReader(image_data),
image_meta['size'], store)
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(
_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum, 'size': size}
try:
image_meta = registry.update_image_metadata(
req.context, image_id, update_data)
except exception.NotFound as e:
msg = _("Image %s could not be found after upload. The image may "
"have been deleted during the upload.") % image_id
LOG.info(msg)
# NOTE(jculp): we need to clean up the datastore if an image
# resource is deleted while the image data is being uploaded
#
# We get "location" from above call to store.add(), any
# exceptions that occur there handle this same issue internally,
# Since this is store-agnostic, should apply to all stores.
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPPreconditionFailed(explanation=msg,
request=req,
content_type='text/plain')
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg,
request=req,
content_type="text/plain")
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = (_("Denying attempt to upload image larger than %d bytes.") %
CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError:
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
msg = _("Received HTTP error while uploading image %s") % image_id
notifier.error('image.upload', msg)
with excutils.save_and_reraise_exception():
LOG.exception(msg)
safe_kill(req, image_id)
except (ValueError, IOError) as e:
msg = _("Client disconnected before sending all data to backend")
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
except Exception as e:
msg = _("Failed to upload image %s") % image_id
LOG.exception(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPInternalServerError(explanation=msg,
request=req,
content_type='text/plain')
return image_meta, location, locations_metadata
def add(self, image_id, image_file, image_size, connection=None):
location = self.create_location(image_id)
if not connection:
connection = self.get_connection(location)
self._create_container_if_missing(location.container, connection)
LOG.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj, image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(int(
math.ceil(float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug(_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container, chunk_name, reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
with excutils.save_and_reraise_exception():
LOG.exception(_("Error during chunked upload to "
"backend, deleting stale chunks"))
self._delete_stale_chunks(connection,
location.container,
written_chunks)
bytes_read = reader.bytes_read
msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s") %
{'chunk_name': chunk_name,
'chunk_id': chunk_id,
'total_chunks': total_chunks,
'bytes_read': bytes_read,
'chunk_etag': chunk_etag})
LOG.debug(msg)
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug(_("Deleting final zero-length chunk"))
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container, location.obj,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag, {})
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
"this location"))
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % {'e': e})
LOG.error(msg)
raise glance.store.BackendException(msg)
def upload(self, req, image_id, data, size):
image_repo = self.gateway.get_repo(req.context)
image = None
try:
image = image_repo.get(image_id)
image.status = 'saving'
try:
image_repo.save(image)
image.set_data(data, size)
image_repo.save(image)
except exception.NotFound as e:
msg = (_("Image %(id)s could not be found after upload."
"The image may have been deleted during the upload: "
"%(error)s Cleaning up the chunks uploaded") % {
'id': image_id,
'error': utils.exception_to_str(e)
})
LOG.warn(msg)
# NOTE(sridevi): Cleaning up the uploaded chunks.
try:
image.delete()
except exception.NotFound:
# NOTE(sridevi): Ignore this exception
pass
raise webob.exc.HTTPGone(explanation=msg,
request=req,
content_type='text/plain')
except ValueError as e:
LOG.debug("Cannot save data for image %(id)s: %(e)s", {
'id': image_id,
'e': utils.exception_to_str(e)
})
self._restore(image_repo, image)
raise webob.exc.HTTPBadRequest(
explanation=utils.exception_to_str(e))
except exception.InvalidImageStatusTransition as e:
msg = utils.exception_to_str(e)
LOG.debug(msg)
raise webob.exc.HTTPConflict(explanation=e.msg, request=req)
except exception.Forbidden as e:
msg = ("Not allowed to upload image data for image %s" % image_id)
LOG.debug(msg)
raise webob.exc.HTTPForbidden(explanation=msg, request=req)
except exception.NotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.msg)
except exception.StorageFull as e:
msg = _("Image storage media "
"is full: %s") % utils.exception_to_str(e)
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.StorageQuotaFull as e:
msg = _("Image exceeds the storage "
"quota: %s") % utils.exception_to_str(e)
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.ImageSizeLimitExceeded as e:
msg = _("The incoming image is "
"too large: %s") % utils.exception_to_str(e)
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image "
"storage media: %s") % utils.exception_to_str(e)
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req)
except webob.exc.HTTPError as e:
with excutils.save_and_reraise_exception():
LOG.error(_("Failed to upload image data due to HTTP error"))
self._restore(image_repo, image)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.exception(
_("Failed to upload image data due to "
"internal error"))
self._restore(image_repo, image)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d and size %d',
image_name, order, image_size)
if image_size == 0:
LOG.warning(_("since image size is zero we will be doing "
"resize-before-write for each chunk which "
"will be considerably slower than normal"))
try:
loc = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
try:
with rbd.Image(ioctx, image_name) as image:
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will
# be slower so setting a higher chunk size may
# speed things up a bit.
if image_size == 0:
length = offset + len(chunk)
LOG.debug(_("resizing image to %s KiB") %
(length / 1024))
image.resize(length)
LOG.debug(_("writing chunk at offset %s") %
(offset))
offset += image.write(chunk, offset)
checksum.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self._delete_image(loc.image, loc.snapshot)
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
db_api = glance.db.get_api()
image_size = image_meta.get('size', None)
try:
remaining = glance.api.common.check_quota(
req.context, image_size, db_api, image_id=image_id)
if remaining is not None:
image_data = utils.LimitingReader(image_data, remaining)
(location,
size,
checksum,
locations_metadata) = glance.store.store_add_to_backend(
image_meta['id'],
utils.CooperativeReader(image_data),
image_meta['size'],
store)
try:
# recheck the quota in case there were simultaneous uploads that
# did not provide the size
glance.api.common.check_quota(
req.context, size, db_api, image_id=image_id)
except exception.StorageQuotaFull:
LOG.info(_('Cleaning up %s after exceeding the quota') % image_id)
glance.store.safe_delete_from_backend(
location, req.context, image_meta['id'])
raise
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = (_("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % {'attr': attr,
'supplied': supplied,
'actual': actual})
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), {'image_id': image_id,
'checksum': checksum,
'size': size})
update_data = {'checksum': checksum,
'size': size}
try:
image_meta = registry.update_image_metadata(req.context,
image_id,
update_data)
except exception.NotFound as e:
msg = _("Image %s could not be found after upload. The image may "
"have been deleted during the upload.") % image_id
LOG.info(msg)
# NOTE(jculp): we need to clean up the datastore if an image
# resource is deleted while the image data is being uploaded
#
# We get "location" from above call to store.add(), any
# exceptions that occur there handle this same issue internally,
# Since this is store-agnostic, should apply to all stores.
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPPreconditionFailed(explanation=msg,
request=req,
content_type='text/plain')
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
# NOTE(dosaboy): do not delete the image since it is likely that this
# conflict is a result of another concurrent upload that will be
# successful.
notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg,
request=req,
content_type="text/plain")
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = (_("Denying attempt to upload image larger than %d bytes.")
% CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageQuotaFull as e:
msg = (_("Denying attempt to upload image because it exceeds the ."
"quota: %s") % e)
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError:
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
msg = _("Received HTTP error while uploading image %s") % image_id
notifier.error('image.upload', msg)
with excutils.save_and_reraise_exception():
LOG.exception(msg)
safe_kill(req, image_id)
except (ValueError, IOError) as e:
msg = _("Client disconnected before sending all data to backend")
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
except Exception as e:
msg = _("Failed to upload image %s") % image_id
LOG.exception(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPInternalServerError(explanation=msg,
request=req,
content_type='text/plain')
return image_meta, location, locations_metadata
def add(self, image_id, image_file, image_size, connection=None):
location = self.create_location(image_id)
if not connection:
connection = self.get_connection(location)
self._create_container_if_missing(location.container, connection)
LOG.debug("Adding image object '%(obj_name)s' "
"to Swift" % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug("Cannot determine image size. Adding as a "
"segmented object to Swift.")
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container,
chunk_name,
reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
with excutils.save_and_reraise_exception():
LOG.exception(
_("Error during chunked upload to "
"backend, deleting stale chunks"))
self._delete_stale_chunks(connection,
location.container,
written_chunks)
bytes_read = reader.bytes_read
msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s" % {
'chunk_name': chunk_name,
'chunk_id': chunk_id,
'total_chunks': total_chunks,
'bytes_read': bytes_read,
'chunk_etag': chunk_etag
})
LOG.debug(msg)
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug("Deleting final zero-length chunk")
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container,
location.obj,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
if swift_store_utils.is_multiple_swift_store_accounts_enabled():
include_creds = False
else:
include_creds = True
return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, {})
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Swift already has an image at "
"this location"))
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %s") % utils.exception_to_str(e))
LOG.error(msg)
raise glance.store.BackendException(msg)
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
db_api = glance.db.get_api()
image_size = image_meta.get('size')
try:
remaining = glance.api.common.check_quota(req.context,
image_size,
db_api,
image_id=image_id)
if remaining is not None:
image_data = utils.LimitingReader(image_data, remaining)
(uri, size, checksum,
location_metadata) = store_api.store_add_to_backend(
image_meta['id'], utils.CooperativeReader(image_data),
image_meta['size'], store)
location_data = {
'url': uri,
'metadata': location_metadata,
'status': 'active'
}
try:
# recheck the quota in case there were simultaneous uploads that
# did not provide the size
glance.api.common.check_quota(req.context,
size,
db_api,
image_id=image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(
_('Cleaning up %s after exceeding '
'the quota') % image_id)
store_utils.safe_delete_from_backend(req.context,
image_meta['id'],
location_data)
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = (_("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % {
'attr': attr,
'supplied': supplied,
'actual': actual
})
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(
"Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d", {
'image_id': image_id,
'checksum': checksum,
'size': size
})
update_data = {'checksum': checksum, 'size': size}
try:
image_meta = registry.update_image_metadata(
req.context, image_id, update_data)
except exception.NotFound as e:
msg = _("Image %s could not be found after upload. The image may "
"have been deleted during the upload.") % image_id
LOG.info(msg)
# NOTE(jculp): we need to clean up the datastore if an image
# resource is deleted while the image data is being uploaded
#
# We get "location_data" from above call to store.add(), any
# exceptions that occur there handle this same issue internally,
# Since this is store-agnostic, should apply to all stores.
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPPreconditionFailed(explanation=msg,
request=req,
content_type='text/plain')
except exception.Duplicate as e:
msg = u"Attempt to upload duplicate image: %s" % e
LOG.debug(msg)
# NOTE(dosaboy): do not delete the image since it is likely that this
# conflict is a result of another concurrent upload that will be
# successful.
notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg,
request=req,
content_type="text/plain")
except exception.Forbidden as e:
msg = u"Forbidden upload attempt: %s" % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % utils.exception_to_str(e)
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = (_("Insufficient permissions on image storage media: %s") %
utils.exception_to_str(e))
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = (_("Denying attempt to upload image larger than %d bytes.") %
CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageQuotaFull as e:
msg = (_("Denying attempt to upload image because it exceeds the ."
"quota: %s") % utils.exception_to_str(e))
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError:
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
msg = _("Received HTTP error while uploading image %s") % image_id
notifier.error('image.upload', msg)
with excutils.save_and_reraise_exception():
LOG.exception(msg)
safe_kill(req, image_id)
except (ValueError, IOError) as e:
msg = "Client disconnected before sending all data to backend"
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
except Exception as e:
msg = _("Failed to upload image %s") % image_id
LOG.exception(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPInternalServerError(explanation=msg,
request=req,
content_type='text/plain')
return image_meta, location_data
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %(name)s with order %(order)d and '
'size %(size)d',
{'name': text_type(image_name),
'order': order,
'size': image_size})
if image_size == 0:
LOG.warning(_("since image size is zero we will be doing "
"resize-before-write for each chunk which "
"will be considerably slower than normal"))
try:
loc = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
try:
with rbd.Image(ioctx, image_name) as image:
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will
# be slower so setting a higher chunk size may
# speed things up a bit.
if image_size == 0:
chunk_length = len(chunk)
length = offset + chunk_length
bytes_written += chunk_length
LOG.debug("resizing image to %s KiB" %
(length / units.Ki))
image.resize(length)
LOG.debug("writing chunk at offset %s" %
(offset))
offset += image.write(chunk, offset)
checksum.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except Exception:
with excutils.save_and_reraise_exception():
# Delete image if one was created
try:
self._delete_image(loc.image, loc.snapshot)
except exception.NotFound:
pass
# Make sure we send back the image size whether provided or inferred.
if image_size == 0:
image_size = bytes_written
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def upload(self, req, image_id, data, size):
image_repo = self.gateway.get_repo(req.context)
try:
image = image_repo.get(image_id)
image.status = 'saving'
try:
image_repo.save(image)
image.set_data(data, size)
image_repo.save(image)
except exception.NotFound as e:
msg = (_("Image %(id)s could not be found after upload."
"The image may have been deleted during the upload: "
"%(error)s Cleaning up the chunks uploaded") %
{'id': image_id,
'error': e})
LOG.warn(msg)
# NOTE(sridevi): Cleaning up the uploaded chunks.
try:
image.delete()
except exception.NotFound:
# NOTE(sridevi): Ignore this exception
pass
raise webob.exc.HTTPGone(explanation=msg,
request=req,
content_type='text/plain')
except ValueError as e:
LOG.debug("Cannot save data for image %(id)s: %(e)s",
{'id': image_id, 'e': utils.exception_to_str(e)})
self._restore(image_repo, image)
raise webob.exc.HTTPBadRequest(explanation=
utils.exception_to_str(e))
except exception.InvalidImageStatusTransition as e:
msg = utils.exception_to_str(e)
LOG.debug(msg)
raise webob.exc.HTTPConflict(explanation=e.msg, request=req)
except exception.Forbidden as e:
msg = ("Not allowed to upload image data for image %s" %
image_id)
LOG.debug(msg)
raise webob.exc.HTTPForbidden(explanation=msg, request=req)
except exception.NotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.msg)
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.StorageQuotaFull as e:
msg = _("Image exceeds the storage quota: %s") % e
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.ImageSizeLimitExceeded as e:
msg = _("The incoming image is too large: %s") % e
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
self._restore(image_repo, image)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req)
except webob.exc.HTTPError as e:
with excutils.save_and_reraise_exception():
LOG.error(_("Failed to upload image data due to HTTP error"))
self._restore(image_repo, image)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.exception(_("Failed to upload image data due to "
"internal error"))
self._restore(image_repo, image)
def open_for_write(self, image_id):
"""
Open a file for writing the image file for an image
with supplied identifier.
:param image_id: Image ID
"""
incomplete_path = self.get_image_filepath(image_id, 'incomplete')
def commit():
with self.get_db() as db:
final_path = self.get_image_filepath(image_id)
LOG.debug(
_("Fetch finished, moving "
"'%(incomplete_path)s' to '%(final_path)s'"),
dict(incomplete_path=incomplete_path,
final_path=final_path))
os.rename(incomplete_path, final_path)
# Make sure that we "pop" the image from the queue...
if self.is_queued(image_id):
os.unlink(self.get_image_filepath(image_id, 'queue'))
filesize = os.path.getsize(final_path)
now = time.time()
db.execute(
"""INSERT INTO cached_images
(image_id, last_accessed, last_modified, hits, size)
VALUES (?, 0, ?, 0, ?)""",
(image_id, now, filesize))
db.commit()
def rollback(e):
with self.get_db() as db:
if os.path.exists(incomplete_path):
invalid_path = self.get_image_filepath(image_id, 'invalid')
LOG.debug(
_("Fetch of cache file failed (%(e)s), rolling "
"back by moving '%(incomplete_path)s' to "
"'%(invalid_path)s'"), {
'e': e,
'incomplete_path': incomplete_path,
'invalid_path': invalid_path
})
os.rename(incomplete_path, invalid_path)
db.execute(
"""DELETE FROM cached_images
WHERE image_id = ?""", (image_id, ))
db.commit()
try:
with open(incomplete_path, 'wb') as cache_file:
yield cache_file
except Exception as e:
with excutils.save_and_reraise_exception():
rollback(e)
else:
commit()
finally:
# if the generator filling the cache file neither raises an
# exception, nor completes fetching all data, neither rollback
# nor commit will have been called, so the incomplete file
# will persist - in that case remove it as it is unusable
# example: ^c from client fetch
if os.path.exists(incomplete_path):
rollback('incomplete fetch')
def rewind(self):
try:
self.data.seek(0)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to rewind image content'))
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %(name)s with order %(order)d and '
'size %(size)d',
{'name': text_type(image_name),
'order': order,
'size': image_size})
if image_size == 0:
LOG.warning(_("since image size is zero we will be doing "
"resize-before-write for each chunk which "
"will be considerably slower than normal"))
try:
loc = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
try:
with rbd.Image(ioctx, image_name) as image:
bytes_written = 0
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
for chunk in chunks:
# If the image size provided is zero we need to do
# a resize for the amount we are writing. This will
# be slower so setting a higher chunk size may
# speed things up a bit.
if image_size == 0:
chunk_length = len(chunk)
length = offset + chunk_length
bytes_written += chunk_length
LOG.debug("resizing image to %s KiB" %
(length / units.Ki))
image.resize(length)
LOG.debug("writing chunk at offset %s" %
(offset))
offset += image.write(chunk, offset)
checksum.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except Exception:
with excutils.save_and_reraise_exception():
# Delete image if one was created
try:
self._delete_image(loc.image, loc.snapshot)
except exception.NotFound:
pass
# Make sure we send back the image size whether provided or inferred.
if image_size == 0:
image_size = bytes_written
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def delete(self, req, id):
"""
Deletes the image and all its chunks from the Glance
:param req: The WSGI/Webob Request object
:param id: The opaque image identifier
:raises HttpBadRequest if image registry is invalid
:raises HttpNotFound if image or any chunk is not available
:raises HttpUnauthorized if image or any chunk is not
deleteable by the requesting user
"""
self._enforce(req, "delete_image")
image = self.get_image_meta_or_404(req, id)
if image["protected"]:
msg = "Image is protected"
LOG.debug(msg)
raise HTTPForbidden(explanation=msg, request=req, content_type="text/plain")
if image["status"] == "pending_delete":
msg = "Forbidden to delete a %s image." % image["status"]
LOG.debug(msg)
raise HTTPForbidden(explanation=msg, request=req, content_type="text/plain")
elif image["status"] == "deleted":
msg = "Image %s not found." % id
LOG.debug(msg)
raise HTTPNotFound(explanation=msg, request=req, content_type="text/plain")
if image["location"] and CONF.delayed_delete:
status = "pending_delete"
else:
status = "deleted"
ori_status = image["status"]
try:
# Update the image from the registry first, since we rely on it
# for authorization checks.
# See https://bugs.launchpad.net/glance/+bug/1065187
image = registry.update_image_metadata(req.context, id, {"status": status})
try:
# The image's location field may be None in the case
# of a saving or queued image, therefore don't ask a backend
# to delete the image if the backend doesn't yet store it.
# See https://bugs.launchpad.net/glance/+bug/747799
if image["location"]:
upload_utils.initiate_deletion(req, image["location"], id, CONF.delayed_delete)
except Exception:
with excutils.save_and_reraise_exception():
registry.update_image_metadata(req.context, id, {"status": ori_status})
registry.delete_image_metadata(req.context, id)
except exception.NotFound as e:
msg = _("Failed to find image to delete: %s") % utils.exception_to_str(e)
for line in msg.split("\n"):
LOG.info(line)
raise HTTPNotFound(explanation=msg, request=req, content_type="text/plain")
except exception.Forbidden as e:
msg = _("Forbidden to delete image: %s") % utils.exception_to_str(e)
for line in msg.split("\n"):
LOG.info(line)
raise HTTPForbidden(explanation=msg, request=req, content_type="text/plain")
else:
self.notifier.info("image.delete", redact_loc(image))
return Response(body="", status=200)
