def add_to_backend(self,
conf,
image_id,
data,
size,
scheme=None,
context=None,
verifier=None):
store_max_size = 7
current_store_size = 2
for location in self.data.keys():
if image_id in location:
raise exception.Duplicate()
if not size:
# 'data' is a string wrapped in a LimitingReader|CooperativeReader
# pipeline, so peek under the hood of those objects to get at the
# string itself.
size = len(data.data.fd)
if (current_store_size + size) > store_max_size:
raise exception.StorageFull()
if context.user == USER2:
raise exception.Forbidden()
if context.user == USER3:
raise exception.StorageWriteDenied()
self.data[image_id] = (data, size)
checksum = 'Z'
return (image_id, size, checksum, self.store_metadata)
def test_catch_error_unhandled_debug_mode(self, mock_function):
mock_function.side_effect = exception.Duplicate()
my_mock = mock.Mock()
my_mock.debug = True
self.assertRaises(exception.Duplicate, cache_manage.list_cached,
my_mock)
def add(self, image_id, image_file, image_size):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({'scheme': self.scheme,
'server_host': self.server_host,
'folder_name': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {'Cookie': cookie, 'Content-Length': image_size}
conn = self._get_http_conn('PUT', loc, headers,
content=image_file)
res = conn.getresponse()
if res.status == httplib.CONFLICT:
raise exception.Duplicate(_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.chunk_size)
if image.exist():
raise exception.Duplicate(_("Sheepdog image %s already exists")
% image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
total = left = image_size
while left > 0:
length = min(self.chunk_size, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def task_create(context, values):
"""Create a task object"""
global DATA
task_values = copy.deepcopy(values)
task_id = task_values.get('id', str(uuid.uuid4()))
required_attributes = ['type', 'status', 'input']
allowed_attributes = ['id', 'type', 'status', 'input', 'result', 'owner',
'message', 'expires_at', 'created_at',
'updated_at', 'deleted_at', 'deleted']
if task_id in DATA['tasks']:
raise exception.Duplicate()
for key in required_attributes:
if key not in task_values:
raise exception.Invalid('%s is a required attribute' % key)
incorrect_keys = set(task_values.keys()) - set(allowed_attributes)
if incorrect_keys:
raise exception.Invalid(
'The keys %s are not valid' % str(incorrect_keys))
task_info_values = _pop_task_info_values(task_values)
task = _task_format(task_id, **task_values)
DATA['tasks'][task_id] = task
task_info = _task_info_create(task['id'], task_info_values)
return _format_task_from_db(task, task_info)
def image_create(context, image_values):
global DATA
image_id = image_values.get('id', str(uuid.uuid4()))
if image_id in DATA['images']:
raise exception.Duplicate()
if 'status' not in image_values:
raise exception.Invalid('status is a required attribute')
allowed_keys = set(['id', 'name', 'status', 'min_ram', 'min_disk', 'size',
'virtual_size', 'checksum', 'locations', 'owner',
'protected', 'is_public', 'container_format',
'disk_format', 'created_at', 'updated_at', 'deleted',
'deleted_at', 'properties', 'tags'])
incorrect_keys = set(image_values.keys()) - allowed_keys
if incorrect_keys:
raise exception.Invalid(
'The keys %s are not valid' % str(incorrect_keys))
image = _image_format(image_id, **image_values)
DATA['images'][image_id] = image
location_data = image_values.get('locations', None)
if location_data is not None:
_image_locations_set(image_id, location_data)
DATA['tags'][image_id] = image.pop('tags', [])
return _normalize_locations(copy.deepcopy(image))
def test_catch_error_unhandled(self, mock_function):
mock_function.side_effect = exception.Duplicate()
my_mock = mock.Mock()
my_mock.debug = False
self.assertEqual(cache_manage.FAILURE,
cache_manage.list_cached(my_mock))
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
loc = StoreLocation({'image_id': image_id})
if self.fs.exists(image_id):
raise exception.Duplicate(
_("GridFS already has an image at "
"location %s") % loc.get_uri())
LOG.debug(
_("Adding a new image to GridFS with id %s and size %s") %
(image_id, image_size))
self.fs.put(image_file, _id=image_id)
image = self._get_file(loc)
LOG.debug(
_("Uploaded image %s, md5 %s, length %s to GridFS") %
(image._id, image.md5, image.length))
return (loc.get_uri(), image.length, image.md5, {})
def _image_update(context, values, image_id, purge_props=False):
"""
Used internally by image_create and image_update
:param context: Request context
:param values: A dict of attributes to set
:param image_id: If None, create the image, otherwise, find and update it
"""
session = get_session()
with session.begin():
# Remove the properties passed in the values mapping. We
# handle properties separately from base image attributes,
# and leaving properties in the values mapping will cause
# a SQLAlchemy model error because SQLAlchemy expects the
# properties attribute of an Image model to be a list and
# not a dict.
properties = values.pop('properties', {})
if image_id:
image_ref = image_get(context, image_id, session=session)
# Perform authorization check
check_mutate_authorization(context, image_ref)
else:
if 'size' in values:
values['size'] = int(values['size'])
if 'min_ram' in values:
values['min_ram'] = int(values['min_ram'])
if 'min_disk' in values:
values['min_disk'] = int(values['min_disk'])
values['is_public'] = bool(values.get('is_public', False))
image_ref = models.Image()
# Need to canonicalize ownership
if 'owner' in values and not values['owner']:
values['owner'] = None
if image_id:
# Don't drop created_at if we're passing it in...
_drop_protected_attrs(models.Image, values)
image_ref.update(values)
# Validate the attributes before we go any further. From my
# investigation, the @validates decorator does not validate
# on new records, only on existing records, which is, well,
# idiotic.
validate_image(image_ref.to_dict())
try:
image_ref.save(session=session)
except IntegrityError, e:
raise exception.Duplicate("Image ID %s already exists!" %
values['id'])
_set_properties_for_image(context, image_ref, properties, purge_props,
session)
def add(self, image_member):
try:
self.get(image_member.member_id)
except exception.NotFound:
pass
else:
msg = _('The target member %(member_id)s is already '
'associated with image %(image_id)s.') % {
'member_id': image_member.member_id,
'image_id': self.image.image_id}
raise exception.Duplicate(msg)
image_member_values = self._format_image_member_to_db(image_member)
# Note(shalq): find the image member including the member marked with
# deleted. We will use only one record to represent membership between
# the same image and member. The record of the deleted image member
# will be reused, if it exists, update its properties instead of
# creating a new one.
members = self.db_api.image_member_find(self.context,
image_id=self.image.image_id,
member=image_member.member_id,
include_deleted=True)
if members:
new_values = self.db_api.image_member_update(self.context,
members[0]['id'],
image_member_values)
else:
new_values = self.db_api.image_member_create(self.context,
image_member_values)
image_member.created_at = new_values['created_at']
image_member.updated_at = new_values['updated_at']
image_member.id = new_values['id']
def schedule_delete_from_backend(uri, conf, context, image_id, **kwargs):
"""
Given a uri and a time, schedule the deletion of an image.
"""
conf.register_opts(delete_opts)
if not conf.delayed_delete:
registry.update_image_metadata(context, image_id,
{'status': 'deleted'})
try:
return delete_from_backend(uri, **kwargs)
except (UnsupportedBackend, exception.NotFound):
msg = _("Failed to delete image from store (%(uri)s).") % locals()
logger.error(msg)
datadir = get_scrubber_datadir(conf)
delete_time = time.time() + conf.scrub_time
file_path = os.path.join(datadir, str(image_id))
utils.safe_mkdirs(datadir)
if os.path.exists(file_path):
msg = _("Image id %(image_id)s already queued for delete") % {
'image_id': image_id
}
raise exception.Duplicate(msg)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.chmod(file_path, 0600)
os.utime(file_path, (delete_time, delete_time))
registry.update_image_metadata(context, image_id,
{'status': 'pending_delete'})
def add_to_backend(self, context, scheme, image_id, data, size):
for location in self.data.keys():
if image_id in location:
raise exception.Duplicate()
self.data[image_id] = (data, size or len(data))
checksum = 'Z'
return (image_id, size, checksum)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
datadir = self._find_best_datadir(image_size)
filepath = os.path.join(datadir, str(image_id))
if os.path.exists(filepath):
raise exception.Duplicate(
_("Image file %s already exists!") % filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError as e:
if e.errno != errno.EACCES:
self._delete_partial(filepath, image_id)
exceptions = {
errno.EFBIG: exception.StorageFull(),
errno.ENOSPC: exception.StorageFull(),
errno.EACCES: exception.StorageWriteDenied()
}
raise exceptions.get(e.errno, e)
except Exception:
with excutils.save_and_reraise_exception():
self._delete_partial(filepath, image_id)
checksum_hex = checksum.hexdigest()
metadata = self._get_metadata()
LOG.debug(
_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s"), {
'bytes_written': bytes_written,
'filepath': filepath,
'checksum_hex': checksum_hex
})
return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size):
"""Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
`glance.common.exception.UnexpectedStatus` if the upload
request returned an unexpected status. The expected responses
are 201 Created and 200 OK.
"""
checksum = hashlib.md5()
image_file = _Reader(image_file, checksum)
loc = StoreLocation({
'scheme': self.scheme,
'server_host': self.server_host,
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id
})
cookie = self._build_vim_cookie_header(
self._session.vim.client.options.transport.cookiejar)
headers = {
'Connection': 'Keep-Alive',
'Cookie': cookie,
'Transfer-Encoding': 'chunked'
}
try:
conn = self._get_http_conn('PUT', loc, headers, content=image_file)
res = conn.getresponse()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(
_('Failed to upload content of image '
'%(image)s') % {'image': image_id})
if res.status == httplib.CONFLICT:
raise exception.Duplicate(
_("Image file %(image_id)s already "
"exists!") % {'image_id': image_id})
if res.status not in (httplib.CREATED, httplib.OK):
msg = (_('Failed to upload content of image %(image)s') % {
'image': image_id
})
LOG.error(msg)
raise exception.UnexpectedStatus(status=res.status,
body=res.read())
return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
filepath = os.path.join(self.datadir, str(image_id))
if os.path.exists(filepath):
raise exception.Duplicate(
_("Image file %s already exists!") % filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
for buf in utils.chunkreadable(image_file,
ChunkedFile.CHUNKSIZE):
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError as e:
if e.errno in [errno.EFBIG, errno.ENOSPC]:
try:
os.unlink(filepath)
except Exception:
msg = _('Unable to remove partial image data for image %s')
LOG.error(msg % image_id)
raise exception.StorageFull()
elif e.errno == errno.EACCES:
raise exception.StorageWriteDenied()
else:
raise
checksum_hex = checksum.hexdigest()
LOG.debug(
_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s") % locals())
return ('file://%s' % filepath, bytes_written, checksum_hex)
def add_to_backend(self, context, scheme, image_id, data, size):
store_max_size = 7
current_store_size = 2
for location in self.data.keys():
if image_id in location:
raise exception.Duplicate()
if size and (current_store_size + size) > store_max_size:
raise exception.StorageFull()
if context.user == USER2:
raise exception.Forbidden()
if context.user == USER3:
raise exception.StorageWriteDenied()
self.data[image_id] = (data, size or len(data))
checksum = 'Z'
return (image_id, size, checksum)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d', image_name, order)
try:
loc = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
try:
with rbd.Image(ioctx, image_name) as image:
offset = 0
chunks = utils.chunkreadable(image_file,
self.chunk_size)
for chunk in chunks:
offset += image.write(chunk, offset)
checksum.update(chunk)
if loc.snapshot:
image.create_snap(loc.snapshot)
image.protect_snap(loc.snapshot)
except:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self._delete_image(loc.image, loc.snapshot)
return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
loc = StoreLocation({'image_id': image_id})
if self.fs.exists(image_id):
raise exception.Duplicate(
_("GridFS already has an image at "
"location %s") % loc.get_uri())
LOG.debug(
_("Adding a new image to GridFS with id %(id)s and "
"size %(size)s") % {
'id': image_id,
'size': image_size
})
try:
self.fs.put(image_file, _id=image_id)
image = self._get_file(loc)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
self.fs.delete(image_id)
LOG.debug(
_("Uploaded image %(id)s, md5 %(md5)s, length %(length)s "
"to GridFS") % {
'id': image._id,
'md5': image.md5,
'length': image.length
})
return (loc.get_uri(), image.length, image.md5, {})
def schedule_delayed_delete_from_backend(uri, image_id, **kwargs):
"""Given a uri, schedule the deletion of an image."""
datadir = CONF.scrubber_datadir
delete_time = time.time() + CONF.scrub_time
file_path = os.path.join(datadir, str(image_id))
utils.safe_mkdirs(datadir)
if os.path.exists(file_path):
msg = _("Image id %(image_id)s already queued for delete") % {
'image_id': image_id
}
raise exception.Duplicate(msg)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.chmod(file_path, 0600)
os.utime(file_path, (delete_time, delete_time))
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
:note By default, the backend writes the image data to a file
`/<DATADIR>/<ID>`, where <DATADIR> is the value of
the filesystem_store_datadir configuration option and <ID>
is the supplied image ID.
"""
filepath = os.path.join(self.datadir, str(image_id))
if os.path.exists(filepath):
raise exception.Duplicate(
_("Image file %s already exists!") % filepath)
checksum = hashlib.md5()
bytes_written = 0
try:
with open(filepath, 'wb') as f:
while True:
buf = image_file.read(ChunkedFile.CHUNKSIZE)
if not buf:
break
bytes_written += len(buf)
checksum.update(buf)
f.write(buf)
except IOError:
raise exception.StorageFull()
checksum_hex = checksum.hexdigest()
logger.debug(
_("Wrote %(bytes_written)d bytes to %(filepath)s with "
"checksum %(checksum_hex)s") % locals())
return ('file://%s' % filepath, bytes_written, checksum_hex)
def add(self, image_member):
try:
self.get(image_member.member_id)
except exception.NotFound:
pass
else:
msg = _('The target member %(member_id)s is already '
'associated with image %(image_id)s.') % {
'member_id': image_member.member_id,
'image_id': self.image.image_id}
raise exception.Duplicate(msg)
image_member_values = self._format_image_member_to_db(image_member)
new_values = self.db_api.image_member_create(self.context,
image_member_values)
image_member.created_at = new_values['created_at']
image_member.updated_at = new_values['updated_at']
image_member.id = new_values['id']
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d', image_name, order)
try:
location = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
if location.snapshot:
image.create_snap(location.snapshot)
image.protect_snap(location.snapshot)
return (location.get_uri(), image_size, checksum.hexdigest())
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.WRITE_CHUNKSIZE)
if image.exist():
raise exception.Duplicate(
_("Sheepdog image %s already exists") % image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
try:
total = left = image_size
while left > 0:
length = min(self.WRITE_CHUNKSIZE, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
image.delete()
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def image_create(context, image_values):
global DATA
image_id = image_values.get('id', uuidutils.generate_uuid())
if image_id in DATA['images']:
raise exception.Duplicate()
if 'status' not in image_values:
raise exception.Invalid('status is a required attribute')
allowed_keys = set(['id', 'name', 'status', 'min_ram', 'min_disk', 'size',
'checksum', 'locations', 'owner', 'protected',
'is_public', 'container_format', 'disk_format',
'created_at', 'updated_at', 'deleted_at', 'deleted',
'properties', 'tags'])
if set(image_values.keys()) - allowed_keys:
raise exception.Invalid()
image = _image_format(image_id, **image_values)
DATA['images'][image_id] = image
DATA['tags'][image_id] = image.pop('tags', [])
return image
def schedule_delete_from_backend(uri, context, image_id, **kwargs):
"""
Given a uri and a time, schedule the deletion of an image.
"""
if not CONF.delayed_delete:
registry.update_image_metadata(context, image_id,
{'status': 'deleted'})
try:
return delete_from_backend(context, uri, **kwargs)
except (UnsupportedBackend, exception.StoreDeleteNotSupported,
exception.NotFound):
exc_type = sys.exc_info()[0].__name__
msg = (_("Failed to delete image at %s from store (%s)") %
(uri, exc_type))
LOG.error(msg)
finally:
# avoid falling through to the delayed deletion logic
return
datadir = CONF.scrubber_datadir
delete_time = time.time() + CONF.scrub_time
file_path = os.path.join(datadir, str(image_id))
utils.safe_mkdirs(datadir)
if os.path.exists(file_path):
msg = _("Image id %(image_id)s already queued for delete") % {
'image_id': image_id
}
raise exception.Duplicate(msg)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.chmod(file_path, 0600)
os.utime(file_path, (delete_time, delete_time))
registry.update_image_metadata(context, image_id,
{'status': 'pending_delete'})
def add(self, image_id, image_file, image_size, connection=None):
location = self.create_location(image_id)
if not connection:
connection = self.get_connection(location)
self._create_container_if_missing(location.container, connection)
LOG.debug(
_("Adding image object '%(obj_name)s' "
"to Swift") % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug(
_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container,
chunk_name,
reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
self._delete_stale_chunks(connection,
location.container,
written_chunks)
raise
bytes_read = reader.bytes_read
msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s")
LOG.debug(msg % locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug(_("Deleting final zero-length chunk"))
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container,
location.obj,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Swift already has an image at "
"this location"))
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
LOG.error(msg)
raise glance.store.BackendException(msg)
def data_iterator():
self.notifier.log = []
yield 'abcde'
raise exception.Duplicate('Cant have duplicates')
def _image_update(context, values, image_id, purge_props=False):
"""
Used internally by image_create and image_update
:param context: Request context
:param values: A dict of attributes to set
:param image_id: If None, create the image, otherwise, find and update it
"""
session = _get_session()
with session.begin():
# Remove the properties passed in the values mapping. We
# handle properties separately from base image attributes,
# and leaving properties in the values mapping will cause
# a SQLAlchemy model error because SQLAlchemy expects the
# properties attribute of an Image model to be a list and
# not a dict.
properties = values.pop('properties', {})
try:
locations = values.pop('locations')
locations_provided = True
except KeyError:
locations_provided = False
if image_id:
image_ref = _image_get(context, image_id, session=session)
# Perform authorization check
_check_mutate_authorization(context, image_ref)
else:
if values.get('size') is not None:
values['size'] = int(values['size'])
if 'min_ram' in values:
values['min_ram'] = int(values['min_ram'] or 0)
if 'min_disk' in values:
values['min_disk'] = int(values['min_disk'] or 0)
values['is_public'] = bool(values.get('is_public', False))
values['protected'] = bool(values.get('protected', False))
image_ref = models.Image()
# Need to canonicalize ownership
if 'owner' in values and not values['owner']:
values['owner'] = None
if image_id:
# Don't drop created_at if we're passing it in...
_drop_protected_attrs(models.Image, values)
#NOTE(iccha-sethi): updated_at must be explicitly set in case
# only ImageProperty table was modifited
values['updated_at'] = timeutils.utcnow()
image_ref.update(values)
# Validate the attributes before we go any further. From my
# investigation, the @validates decorator does not validate
# on new records, only on existing records, which is, well,
# idiotic.
values = _validate_image(image_ref.to_dict())
_update_values(image_ref, values)
try:
image_ref.save(session=session)
except sqlalchemy.exc.IntegrityError:
raise exception.Duplicate("Image ID %s already exists!"
% values['id'])
_set_properties_for_image(context, image_ref, properties, purge_props,
session)
if locations_provided:
_image_locations_set(image_ref.id, locations, session)
return image_get(context, image_ref.id)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:note Swift cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Swift and then write an manifest
to Swift that contains information about the chunks.
This same chunking process is used by default for images
of an unknown size, as pushing them directly to swift would
fail if the image turns out to be greater than 5GB.
"""
swift_conn = self._make_swift_connection(self.full_auth_address,
self.user,
self.key,
storage_url=self.storage_url,
token=self.token)
obj_name = str(image_id)
if self.multi_tenant:
# NOTE: When using multi-tenant we create containers for each
# image so we can set permissions on each image in swift
container = self.container + '_' + obj_name
auth_or_store_url = self.storage_url
else:
container = self.container
auth_or_store_url = self.auth_address
create_container_if_missing(container, swift_conn)
location = StoreLocation({
'scheme': self.scheme,
'container': container,
'obj': obj_name,
'auth_or_store_url': auth_or_store_url,
'user': self.user,
'key': self.key
})
LOG.debug(
_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = swift_conn.put_object(container,
obj_name,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug(
_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (obj_name, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
chunk_etag = swift_conn.put_object(
container,
chunk_name,
reader,
content_length=content_length)
bytes_read = reader.bytes_read
msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s")
LOG.debug(msg % locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if image_size == 0.
LOG.debug(_("Deleting final zero-length chunk"))
swift_conn.delete_object(container, chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the image_size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (container, obj_name)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
swift_conn.put_object(container,
obj_name,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swiftclient.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Swift already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
LOG.error(msg)
raise glance.store.BackendException(msg)
def _do_request(self, method, url, body, headers):
"""
Connects to the server and issues a request. Handles converting
any returned HTTP error status codes to OpenStack/Glance exceptions
and closing the server connection. Returns the result data, or
raises an appropriate exception.
:param method: HTTP method ("GET", "POST", "PUT", etc...)
:param url: urlparse.ParsedResult object with URL information
:param body: data to send (as string, filelike or iterable),
or None (default)
:param headers: mapping of key/value pairs to add as headers
:note
If the body param has a read attribute, and method is either
POST or PUT, this method will automatically conduct a chunked-transfer
encoding and use the body as a file object or iterable, transferring
chunks of data using the connection's send() method. This allows large
objects to be transferred efficiently without buffering the entire
body in memory.
"""
if url.query:
path = url.path + "?" + url.query
else:
path = url.path
try:
connection_type = self.get_connection_type()
headers = self._encode_headers(headers or {})
if 'x-auth-token' not in headers and self.auth_tok:
headers['x-auth-token'] = self.auth_tok
c = connection_type(url.hostname, url.port, **self.connect_kwargs)
def _pushing(method):
return method.lower() in ('post', 'put')
def _simple(body):
return body is None or isinstance(body, six.string_types)
def _filelike(body):
return hasattr(body, 'read')
def _sendbody(connection, iter):
connection.endheaders()
for sent in iter:
# iterator has done the heavy lifting
pass
def _chunkbody(connection, iter):
connection.putheader('Transfer-Encoding', 'chunked')
connection.endheaders()
for chunk in iter:
connection.send('%x\r\n%s\r\n' % (len(chunk), chunk))
connection.send('0\r\n\r\n')
# Do a simple request or a chunked request, depending
# on whether the body param is file-like or iterable and
# the method is PUT or POST
#
if not _pushing(method) or _simple(body):
# Simple request...
c.request(method, path, body, headers)
elif _filelike(body) or self._iterable(body):
c.putrequest(method, path)
use_sendfile = self._sendable(body)
# According to HTTP/1.1, Content-Length and Transfer-Encoding
# conflict.
for header, value in headers.items():
if use_sendfile or header.lower() != 'content-length':
c.putheader(header, str(value))
iter = self.image_iterator(c, headers, body)
if use_sendfile:
# send actual file without copying into userspace
_sendbody(c, iter)
else:
# otherwise iterate and chunk
_chunkbody(c, iter)
else:
raise TypeError('Unsupported image type: %s' % body.__class__)
res = c.getresponse()
def _retry(res):
return res.getheader('Retry-After')
status_code = self.get_status_code(res)
if status_code in self.OK_RESPONSE_CODES:
return res
elif status_code in self.REDIRECT_RESPONSE_CODES:
raise exception.RedirectException(res.getheader('Location'))
elif status_code == httplib.UNAUTHORIZED:
raise exception.NotAuthenticated(res.read())
elif status_code == httplib.FORBIDDEN:
raise exception.Forbidden(res.read())
elif status_code == httplib.NOT_FOUND:
raise exception.NotFound(res.read())
elif status_code == httplib.CONFLICT:
raise exception.Duplicate(res.read())
elif status_code == httplib.BAD_REQUEST:
raise exception.Invalid(res.read())
elif status_code == httplib.MULTIPLE_CHOICES:
raise exception.MultipleChoices(body=res.read())
elif status_code == httplib.REQUEST_ENTITY_TOO_LARGE:
raise exception.LimitExceeded(retry=_retry(res),
body=res.read())
elif status_code == httplib.INTERNAL_SERVER_ERROR:
raise exception.ServerError()
elif status_code == httplib.SERVICE_UNAVAILABLE:
raise exception.ServiceUnavailable(retry=_retry(res))
else:
raise exception.UnexpectedStatus(status=status_code,
body=res.read())
except (socket.error, IOError) as e:
raise exception.ClientConnectionError(e)
