def test_cooperative_reader_of_iterator(self):
"""Ensure cooperative reader supports iterator backends too"""
data = b'abcdefgh'
data_list = [data[i:i + 1] * 3 for i in range(len(data))]
reader = utils.CooperativeReader(data_list)
chunks = []
while True:
chunks.append(reader.read(3))
if chunks[-1] == b'':
break
meat = b''.join(chunks)
self.assertEqual(b'aaabbbcccdddeeefffggghhh', meat)
def set_data(self, data, size=None):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
# Create the verifier for signature verification (if correct properties
# are present)
extra_props = self.image.extra_properties
if (signature_utils.should_create_verifier(extra_props)):
# NOTE(bpoulos): if creating verifier fails, exception will be
# raised
img_signature = extra_props[signature_utils.SIGNATURE]
hash_method = extra_props[signature_utils.HASH_METHOD]
key_type = extra_props[signature_utils.KEY_TYPE]
cert_uuid = extra_props[signature_utils.CERT_UUID]
verifier = signature_utils.get_verifier(
context=self.context,
img_signature_certificate_uuid=cert_uuid,
img_signature_hash_method=hash_method,
img_signature=img_signature,
img_signature_key_type=key_type)
else:
verifier = None
location, size, checksum, loc_meta = self.store_api.add_to_backend(
CONF,
self.image.image_id,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap),
size,
context=self.context,
verifier=verifier)
# NOTE(bpoulos): if verification fails, exception will be raised
if verifier:
try:
verifier.verify()
LOG.info(_LI("Successfully verified signature for image %s"),
self.image.image_id)
except crypto_exception.InvalidSignature:
self.store_api.delete_from_backend(location,
context=self.context)
raise cursive_exception.SignatureVerificationError(
_('Signature verification failed'))
self.image.locations = [{
'url': location,
'metadata': loc_meta,
'status': 'active'
}]
self.image.size = size
self.image.checksum = checksum
self.image.status = 'active'
def upload_to_store(self, data, size):
if size is None: # NOTE(ativelkov): None is "unknown size"
size = 0
location, ret_size, checksum, loc_meta = self.store_api.add_to_backend(
CONF,
self.blob.item_key,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap),
size,
context=self.context)
self.blob.size = ret_size
self.blob.locations = [{'status': 'active', 'value': location}]
self.blob.checksum = checksum
def _test_reader_chunked(self, chunk_size, read_size, max_iterations=5):
generator = self._create_generator(chunk_size, max_iterations)
reader = utils.CooperativeReader(generator)
result = bytearray()
while True:
data = reader.read(read_size)
if len(data) == 0:
break
self.assertLessEqual(len(data), read_size)
result += data
expected = (b'a' * chunk_size + b'b' * chunk_size + b'c' * chunk_size +
b'a' * chunk_size + b'b' * chunk_size)
self.assertEqual(expected, bytes(result))
def test_cooperative_reader_on_iterator_with_buffer(self):
"""Ensure cooperative reader is happy with empty iterators"""
data_list = [b'abcd', b'efgh']
reader = utils.CooperativeReader(data_list)
# read from part of a chunk, get the first item into the buffer
self.assertEqual(b'ab', reader.read(2))
# read purely from buffer
self.assertEqual(b'c', reader.read(1))
# unbounded read grabs the rest of the buffer
self.assertEqual(b'd', reader.read())
# then the whole next chunk
self.assertEqual(b'efgh', reader.read())
# past that, it's always empty
self.assertEqual(b'', reader.read())
def test_cooperative_reader(self):
"""Ensure cooperative reader class accesses all bytes of file"""
BYTES = 1024
bytes_read = 0
with tempfile.TemporaryFile('w+') as tmp_fd:
tmp_fd.write('*' * BYTES)
tmp_fd.seek(0)
for chunk in utils.CooperativeReader(tmp_fd):
bytes_read += len(chunk)
self.assertEqual(BYTES, bytes_read)
bytes_read = 0
with tempfile.TemporaryFile('w+') as tmp_fd:
tmp_fd.write('*' * BYTES)
tmp_fd.seek(0)
reader = utils.CooperativeReader(tmp_fd)
byte = reader.read(1)
while len(byte) != 0:
bytes_read += 1
byte = reader.read(1)
self.assertEqual(BYTES, bytes_read)
def set_data(self, data, size=None):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
location, size, checksum, loc_meta = self.store_api.add_to_backend(
self.context, CONF.default_store, self.image.image_id,
utils.CooperativeReader(data), size)
self.image.locations = [{
'url': location,
'metadata': loc_meta,
'status': 'active'
}]
self.image.size = size
self.image.checksum = checksum
self.image.status = 'active'
def set_data(self, data, size=None):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
location, size, checksum, loc_meta = self.store_api.add_to_backend(
CONF,
self.image.image_id,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap),
size,
context=self.context)
loc_meta = loc_meta or {}
loc_meta['is_default'] = 'true'
self.image.locations = [{'url': location, 'metadata': loc_meta,
'status': 'active'}]
self.image.size = size
self.image.checksum = checksum
self.image.status = 'active'
def set_data(self, data, size=None):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
# Create the verifier for signature verification (if correct properties
# are present)
if (signature_utils.should_create_verifier(
self.image.extra_properties)):
# NOTE(bpoulos): if creating verifier fails, exception will be
# raised
verifier = signature_utils.get_verifier(
self.context, self.image.extra_properties)
else:
verifier = None
location, size, checksum, loc_meta = self.store_api.add_to_backend(
CONF,
self.image.image_id,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap),
size,
context=self.context,
verifier=verifier)
self._verify_signature_if_needed(checksum)
# NOTE(bpoulos): if verification fails, exception will be raised
if verifier:
try:
verifier.verify()
LOG.info(_LI("Successfully verified signature for image %s"),
self.image.image_id)
except crypto_exception.InvalidSignature:
raise exception.SignatureVerificationError(
_('Signature verification failed'))
self.image.locations = [{
'url': location,
'metadata': loc_meta,
'status': 'active'
}]
self.image.size = size
self.image.checksum = checksum
self.image.status = 'active'
def stage(self, req, image_id, data, size):
image_repo = self.gateway.get_repo(req.context)
image = None
# NOTE(jokke): this is horrible way to do it but as long as
# glance_store is in a shape it is, the only way. Don't hold me
# accountable for it.
# TODO(abhishekk): After removal of backend module from glance_store
# need to change this to use multi_backend module.
def _build_staging_store():
conf = cfg.ConfigOpts()
try:
backend.register_opts(conf)
except cfg.DuplicateOptError:
pass
conf.set_override('filesystem_store_datadir',
CONF.node_staging_uri[7:],
group='glance_store')
staging_store = backend._load_store(conf, 'file')
try:
staging_store.configure()
except AttributeError:
msg = _("'node_staging_uri' is not set correctly. Could not "
"load staging store.")
raise exception.BadStoreUri(message=msg)
return staging_store
staging_store = _build_staging_store()
try:
image = image_repo.get(image_id)
image.status = 'uploading'
image_repo.save(image, from_state='queued')
try:
staging_store.add(
image_id, utils.LimitingReader(
utils.CooperativeReader(data), CONF.image_size_cap), 0)
except glance_store.Duplicate as e:
msg = _("The image %s has data on staging") % image_id
raise webob.exc.HTTPConflict(explanation=msg)
except exception.NotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.msg)
except glance_store.StorageFull as e:
msg = _("Image storage media "
"is full: %s") % encodeutils.exception_to_unicode(e)
LOG.error(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.StorageQuotaFull as e:
msg = _("Image exceeds the storage "
"quota: %s") % encodeutils.exception_to_unicode(e)
LOG.debug(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.ImageSizeLimitExceeded as e:
msg = _("The incoming image is "
"too large: %s") % encodeutils.exception_to_unicode(e)
LOG.debug(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except glance_store.StorageWriteDenied as e:
msg = _("Insufficient permissions on image "
"storage media: %s") % encodeutils.exception_to_unicode(e)
LOG.error(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req)
except exception.InvalidImageStatusTransition as e:
msg = encodeutils.exception_to_unicode(e)
LOG.debug(msg)
raise webob.exc.HTTPConflict(explanation=e.msg, request=req)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to stage image data due to "
"internal error"))
self._restore(image_repo, image)
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
db_api = glance.db.get_api()
image_size = image_meta.get('size')
try:
remaining = glance.api.common.check_quota(req.context,
image_size,
db_api,
image_id=image_id)
if remaining is not None:
image_data = utils.LimitingReader(image_data, remaining)
(uri, size, checksum,
location_metadata) = store_api.store_add_to_backend(
image_meta['id'], utils.CooperativeReader(image_data),
image_meta['size'], store)
location_data = {
'url': uri,
'metadata': location_metadata,
'status': 'active'
}
try:
# recheck the quota in case there were simultaneous uploads that
# did not provide the size
glance.api.common.check_quota(req.context,
size,
db_api,
image_id=image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(
_('Cleaning up %s after exceeding '
'the quota') % image_id)
store_utils.safe_delete_from_backend(req.context,
image_meta['id'],
location_data)
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = (_("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % {
'attr': attr,
'supplied': supplied,
'actual': actual
})
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(
"Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d", {
'image_id': image_id,
'checksum': checksum,
'size': size
})
update_data = {'checksum': checksum, 'size': size}
try:
image_meta = registry.update_image_metadata(
req.context, image_id, update_data)
except exception.NotFound as e:
msg = _("Image %s could not be found after upload. The image may "
"have been deleted during the upload.") % image_id
LOG.info(msg)
# NOTE(jculp): we need to clean up the datastore if an image
# resource is deleted while the image data is being uploaded
#
# We get "location_data" from above call to store.add(), any
# exceptions that occur there handle this same issue internally,
# Since this is store-agnostic, should apply to all stores.
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPPreconditionFailed(explanation=msg,
request=req,
content_type='text/plain')
except exception.Duplicate as e:
msg = u"Attempt to upload duplicate image: %s" % e
LOG.debug(msg)
# NOTE(dosaboy): do not delete the image since it is likely that this
# conflict is a result of another concurrent upload that will be
# successful.
notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg,
request=req,
content_type="text/plain")
except exception.Forbidden as e:
msg = u"Forbidden upload attempt: %s" % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % utils.exception_to_str(e)
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = (_("Insufficient permissions on image storage media: %s") %
utils.exception_to_str(e))
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = (_("Denying attempt to upload image larger than %d bytes.") %
CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageQuotaFull as e:
msg = (_("Denying attempt to upload image because it exceeds the ."
"quota: %s") % utils.exception_to_str(e))
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError:
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
msg = _("Received HTTP error while uploading image %s") % image_id
notifier.error('image.upload', msg)
with excutils.save_and_reraise_exception():
LOG.exception(msg)
safe_kill(req, image_id)
except (ValueError, IOError) as e:
msg = "Client disconnected before sending all data to backend"
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
except Exception as e:
msg = _("Failed to upload image %s") % image_id
LOG.exception(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPInternalServerError(explanation=msg,
request=req,
content_type='text/plain')
return image_meta, location_data
def _upload(self, req, image_meta):
"""
Uploads the payload of the request to a backend store in
Glance. If the `x-image-meta-store` header is set, Glance
will attempt to use that scheme; if not, Glance will use the
scheme set by the flag `default_store` to find the backing store.
:param req: The WSGI/Webob Request object
:param image_meta: Mapping of metadata about image
:raises HTTPConflict if image already exists
:retval The location where the image was stored
"""
copy_from = self._copy_from(req)
if copy_from:
try:
image_data, image_size = self._get_from_store(req.context,
copy_from)
except Exception as e:
self._safe_kill(req, image_meta['id'])
msg = _("Copy from external source failed: %s") % e
LOG.debug(msg)
return
image_meta['size'] = image_size or image_meta['size']
else:
try:
req.get_content_type('application/octet-stream')
except exception.InvalidContentType:
self._safe_kill(req, image_meta['id'])
msg = _("Content-Type must be application/octet-stream")
LOG.debug(msg)
raise HTTPBadRequest(explanation=msg)
image_data = req.body_file
scheme = req.headers.get('x-image-meta-store', CONF.default_store)
store = self.get_store_or_400(req, scheme)
image_id = image_meta['id']
LOG.debug(_("Setting image %s to status 'saving'"), image_id)
registry.update_image_metadata(req.context, image_id,
{'status': 'saving'})
LOG.debug(_("Uploading image data for image %(image_id)s "
"to %(scheme)s store"), locals())
try:
self.notifier.info("image.prepare", redact_loc(image_meta))
location, size, checksum = store.add(
image_meta['id'],
utils.CooperativeReader(image_data),
image_meta['size'])
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
self._safe_kill(req, image_id)
self._initiate_deletion(req, location, image_id)
raise HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum,
'size': size}
image_meta = registry.update_image_metadata(req.context,
image_id,
update_data)
self.notifier.info('image.upload', redact_loc(image_meta))
return location
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
self._safe_kill(req, image_id)
raise HTTPConflict(explanation=msg, request=req)
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
self._safe_kill(req, image_id)
raise HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPServiceUnavailable(explanation=msg, request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = _("Denying attempt to upload image larger than %d bytes."
% CONF.image_size_cap)
LOG.info(msg)
self._safe_kill(req, image_id)
raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
content_type='text/plain')
except HTTPError as e:
self._safe_kill(req, image_id)
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
raise e
except Exception as e:
LOG.exception(_("Failed to upload image"))
self._safe_kill(req, image_id)
raise HTTPInternalServerError(request=req)
def _upload(self, req, image_meta):
"""
Uploads the payload of the request to a backend store in
Glance. If the `x-image-meta-store` header is set, Glance
will attempt to use that scheme; if not, Glance will use the
scheme set by the flag `default_store` to find the backing store.
:param req: The WSGI/Webob Request object
:param image_meta: Mapping of metadata about image
:raises HTTPConflict if image already exists
:retval The location where the image was stored
"""
copy_from = self._copy_from(req)
if copy_from:
image_data, image_size = self._get_from_store(
req.context, copy_from)
image_meta['size'] = image_size or image_meta['size']
else:
try:
req.get_content_type('application/octet-stream')
except exception.InvalidContentType:
self._safe_kill(req, image_meta['id'])
msg = _("Content-Type must be application/octet-stream")
LOG.error(msg)
raise HTTPBadRequest(explanation=msg)
image_data = req.body_file
scheme = req.headers.get('x-image-meta-store', CONF.default_store)
store = self.get_store_or_400(req, scheme)
image_id = image_meta['id']
LOG.debug(_("Setting image %s to status 'saving'"), image_id)
registry.update_image_metadata(req.context, image_id,
{'status': 'saving'})
LOG.debug(
_("Uploading image data for image %(image_id)s "
"to %(scheme)s store"), locals())
try:
location, size, checksum = store.add(
image_meta['id'], utils.CooperativeReader(image_data),
image_meta['size'])
# Verify any supplied checksum value matches checksum
# returned from store when adding image
supplied_checksum = image_meta.get('checksum')
if supplied_checksum and supplied_checksum != checksum:
msg = _("Supplied checksum (%(supplied_checksum)s) and "
"checksum generated from uploaded image "
"(%(checksum)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
self._safe_kill(req, image_id)
raise HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Update the database with the checksum returned
# from the backend store
LOG.debug(
_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum, 'size': size}
image_meta = registry.update_image_metadata(
req.context, image_id, update_data)
self.notifier.info('image.upload', image_meta)
return location
except exception.Duplicate, e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPConflict(explanation=msg, request=req)
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
try:
(location, size, checksum,
locations_metadata) = glance.store.store_add_to_backend(
image_meta['id'], utils.CooperativeReader(image_data),
image_meta['size'], store)
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(
_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum, 'size': size}
try:
image_meta = registry.update_image_metadata(
req.context, image_id, update_data)
except exception.NotFound as e:
msg = _("Image %s could not be found after upload. The image may "
"have been deleted during the upload.") % image_id
LOG.info(msg)
# NOTE(jculp): we need to clean up the datastore if an image
# resource is deleted while the image data is being uploaded
#
# We get "location" from above call to store.add(), any
# exceptions that occur there handle this same issue internally,
# Since this is store-agnostic, should apply to all stores.
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPPreconditionFailed(explanation=msg,
request=req,
content_type='text/plain')
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg,
request=req,
content_type="text/plain")
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = (_("Denying attempt to upload image larger than %d bytes.") %
CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError:
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
msg = _("Received HTTP error while uploading image %s") % image_id
notifier.error('image.upload', msg)
with excutils.save_and_reraise_exception():
LOG.exception(msg)
safe_kill(req, image_id)
except (ValueError, IOError) as e:
msg = _("Client disconnected before sending all data to backend")
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
except Exception as e:
msg = _("Failed to upload image %s") % image_id
LOG.exception(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPInternalServerError(explanation=msg,
request=req,
content_type='text/plain')
return image_meta, location, locations_metadata
def test_cooperative_reader_unbounded_read_on_empty_iterator(self):
"""Ensure cooperative reader is happy with empty iterators"""
reader = utils.CooperativeReader([])
self.assertEqual(b'', reader.read())
def stage(self, req, image_id, data, size):
try:
ks_quota.enforce_image_staging_total(req.context,
req.context.owner)
except exception.LimitExceeded as e:
raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e),
request=req)
image_repo = self.gateway.get_repo(req.context,
authorization_layer=False)
# NOTE(abhishekk): stage API call does not have its own policy but
# it requires get_image access, this is the right place to check
# whether user has access to image or not
try:
image = image_repo.get(image_id)
except exception.NotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.msg)
api_pol = api_policy.ImageAPIPolicy(req.context,
image,
enforcer=self.policy)
try:
api_pol.modify_image()
except exception.Forbidden as e:
# NOTE(abhishekk): This will throw Forbidden if S-RBAC is not
# enabled
raise webob.exc.HTTPForbidden(explanation=e.msg)
# NOTE(jokke): this is horrible way to do it but as long as
# glance_store is in a shape it is, the only way. Don't hold me
# accountable for it.
# TODO(abhishekk): After removal of backend module from glance_store
# need to change this to use multi_backend module.
def _build_staging_store():
conf = cfg.ConfigOpts()
try:
backend.register_opts(conf)
except cfg.DuplicateOptError:
pass
conf.set_override('filesystem_store_datadir',
CONF.node_staging_uri[7:],
group='glance_store')
staging_store = backend._load_store(conf, 'file')
try:
staging_store.configure()
except AttributeError:
msg = _("'node_staging_uri' is not set correctly. Could not "
"load staging store.")
raise exception.BadStoreUri(message=msg)
return staging_store
# NOTE(abhishekk): Use reserved 'os_glance_staging_store' for staging
# the data, the else part will be removed once multiple backend feature
# is declared as stable.
if CONF.enabled_backends:
staging_store = glance_store.get_store_from_store_identifier(
'os_glance_staging_store')
else:
staging_store = _build_staging_store()
try:
image.status = 'uploading'
image_repo.save(image, from_state='queued')
ks_quota.enforce_image_count_uploading(req.context,
req.context.owner)
try:
uri, size, id, store_info = staging_store.add(
image_id,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap), 0)
image.size = size
except glance_store.Duplicate:
msg = _("The image %s has data on staging") % image_id
raise webob.exc.HTTPConflict(explanation=msg)
# NOTE(danms): Record this worker's
# worker_self_reference_url in the image metadata so we
# know who has the staging data.
self_url = CONF.worker_self_reference_url or CONF.public_endpoint
if self_url:
image.extra_properties['os_glance_stage_host'] = self_url
image_repo.save(image, from_state='uploading')
except exception.NotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.msg)
except glance_store.StorageFull as e:
msg = _("Image storage media "
"is full: %s") % encodeutils.exception_to_unicode(e)
LOG.error(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.StorageQuotaFull as e:
msg = _("Image exceeds the storage "
"quota: %s") % encodeutils.exception_to_unicode(e)
LOG.debug(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.ImageSizeLimitExceeded as e:
msg = _("The incoming image is "
"too large: %s") % encodeutils.exception_to_unicode(e)
LOG.debug(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.LimitExceeded as e:
LOG.debug(str(e))
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e),
request=req)
except glance_store.StorageWriteDenied as e:
msg = _("Insufficient permissions on image "
"storage media: %s") % encodeutils.exception_to_unicode(e)
LOG.error(msg)
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req)
except exception.InvalidImageStatusTransition as e:
msg = encodeutils.exception_to_unicode(e)
LOG.debug(msg)
raise webob.exc.HTTPConflict(explanation=e.msg, request=req)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE("Failed to stage image data due to "
"internal error"))
self._restore(image_repo, image)
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
try:
location, size, checksum = store.add(
image_meta['id'], utils.CooperativeReader(image_data),
image_meta['size'])
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(
_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum, 'size': size}
image_meta = registry.update_image_metadata(req.context, image_id,
update_data)
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPConflict(explanation=msg, request=req)
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = _("Denying attempt to upload image larger than %d bytes." %
CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError as e:
LOG.exception(_("Received HTTP error while uploading image."))
safe_kill(req, image_id)
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
raise e
except Exception as e:
LOG.exception(_("Failed to upload image"))
safe_kill(req, image_id)
raise webob.exc.HTTPInternalServerError(request=req)
return image_meta, location