def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.chunk_size)
if image.exist():
raise exception.Duplicate(_("Sheepdog image %s already exists")
% image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
total = left = image_size
while left > 0:
length = min(self.chunk_size, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest())
def get(self, location, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
try:
resp_headers, resp_body = connection.get_object(
container=location.container, obj=location.obj,
resp_chunk_size=self.CHUNKSIZE)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
class ResponseIndexable(glance.store.Indexable):
def another(self):
try:
return self.wrapped.next()
except StopIteration:
return ''
length = int(resp_headers.get('content-length', 0))
return (ResponseIndexable(resp_body, length), length)
def set_acls(self, location, public=False, read_tenants=None,
write_tenants=None, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
if read_tenants is None:
read_tenants = []
if write_tenants is None:
write_tenants = []
headers = {}
if public:
headers['X-Container-Read'] = ".r:*"
elif read_tenants:
headers['X-Container-Read'] = ','.join(read_tenants)
else:
headers['X-Container-Read'] = ''
write_tenants.extend(self.admin_tenants)
if write_tenants:
headers['X-Container-Write'] = ','.join(write_tenants)
else:
headers['X-Container-Write'] = ''
try:
connection.post_container(location.container, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
def set_acls(self, location, public=False, read_tenants=None,
write_tenants=None, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
if read_tenants is None:
read_tenants = []
if write_tenants is None:
write_tenants = []
headers = {}
if public:
headers['X-Container-Read'] = ".r:*"
elif read_tenants:
headers['X-Container-Read'] = ','.join(read_tenants)
else:
headers['X-Container-Read'] = ''
write_tenants.extend(self.admin_tenants)
if write_tenants:
headers['X-Container-Write'] = ','.join(write_tenants)
else:
headers['X-Container-Write'] = ''
try:
connection.post_container(location.container, headers=headers)
except swiftclient.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
location = StoreLocation({"image": image_id})
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
logger.debug("creating image %s with order %d", image_name, order)
try:
rbd.RBD().create(ioctx, image_name, image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(_("RBD image %s already exists") % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest())
def get(self, location, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
try:
resp_headers, resp_body = connection.get_object(
container=location.container,
obj=location.obj,
resp_chunk_size=self.CHUNKSIZE)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
class ResponseIndexable(glance.store.Indexable):
def another(self):
try:
return self.wrapped.next()
except StopIteration:
return ''
length = int(resp_headers.get('content-length', 0))
return (ResponseIndexable(resp_body, length), length)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.chunk_size)
if image.exist():
raise exception.Duplicate(_("Sheepdog image %s already exists")
% image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
total = left = image_size
while left > 0:
length = min(self.chunk_size, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def get(self, location, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
try:
resp_headers, resp_body = connection.get_object(
container=location.container, obj=location.obj,
resp_chunk_size=self.CHUNKSIZE)
except swiftclient.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
def get(self, location, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
try:
resp_headers, resp_body = connection.get_object(
container=location.container, obj=location.obj,
resp_chunk_size=self.CHUNKSIZE)
except swiftclient.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
raise exception.NotFound(_("Swift could not find image at "
"uri %(uri)s") % locals())
else:
raise
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d',
image_name, order)
try:
location = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
if location.snapshot:
image.create_snap(location.snapshot)
image.protect_snap(location.snapshot)
return (location.get_uri(), image_size, checksum.hexdigest())
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d', image_name, order)
try:
location = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
bytes_left = image_size
while bytes_left > 0:
length = min(self.chunk_size, bytes_left)
data = image_file.read(length)
image.write(data, image_size - bytes_left)
bytes_left -= length
checksum.update(data)
if location.snapshot:
image.create_snap(location.snapshot)
image.protect_snap(location.snapshot)
return (location.get_uri(), image_size, checksum.hexdigest())
def delete(self, location, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
try:
# We request the manifest for the object. If one exists,
# that means the object was uploaded in chunks/segments,
# and we need to delete all the chunks as well as the
# manifest.
manifest = None
try:
headers = connection.head_object(
location.container, location.obj)
manifest = headers.get('x-object-manifest')
except swiftclient.ClientException as e:
if e.http_status != httplib.NOT_FOUND:
raise
if manifest:
# Delete all the chunks before the object manifest itself
obj_container, obj_prefix = manifest.split('/', 1)
segments = connection.get_container(
obj_container, prefix=obj_prefix)[1]
for segment in segments:
# TODO(jaypipes): This would be an easy area to parallelize
# since we're simply sending off parallelizable requests
# to Swift to delete stuff. It's not like we're going to
# be hogging up network or file I/O here...
connection.delete_object(obj_container,
segment['name'])
# Delete object (or, in segmented case, the manifest)
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.WRITE_CHUNKSIZE)
if image.exist():
raise exception.Duplicate(
_("Sheepdog image %s already exists") % image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
try:
total = left = image_size
while left > 0:
length = min(self.WRITE_CHUNKSIZE, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exception.
with excutils.save_and_reraise_exception():
image.delete()
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, checksum
and a dictionary with storage system specific information
:raises `glance.common.exception.Duplicate` if the image already
existed
"""
checksum = hashlib.md5()
image_name = str(image_id)
with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
fsid = None
if hasattr(conn, 'get_fsid'):
fsid = conn.get_fsid()
with conn.open_ioctx(self.pool) as ioctx:
order = int(math.log(self.chunk_size, 2))
LOG.debug('creating image %s with order %d', image_name, order)
try:
location = self._create_image(fsid, ioctx, image_name,
image_size, order)
except rbd.ImageExists:
raise exception.Duplicate(
_('RBD image %s already exists') % image_id)
with rbd.Image(ioctx, image_name) as image:
offset = 0
chunks = utils.chunkreadable(image_file, self.chunk_size)
for chunk in chunks:
offset += image.write(chunk, offset)
checksum.update(chunk)
if location.snapshot:
image.create_snap(location.snapshot)
image.protect_snap(location.snapshot)
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval tuple of URL in backing store, bytes written, and checksum
:raises `glance.store.exceptions.Duplicate` if the image already
existed
"""
image = SheepdogImage(self.addr, self.port, image_id,
self.chunk_size)
if image.exist():
raise exceptions.Duplicate(_("Sheepdog image %s already exists")
% image_id)
location = StoreLocation({'image': image_id})
checksum = hashlib.md5()
image.create(image_size)
try:
total = left = image_size
while left > 0:
length = min(self.chunk_size, left)
data = image_file.read(length)
image.write(data, total - left, length)
left -= length
checksum.update(data)
except Exception:
# Note(zhiyan): clean up already received data when
# error occurs such as ImageSizeLimitExceeded exceptions.
with excutils.save_and_reraise_exception():
image.delete()
return (location.get_uri(), image_size, checksum.hexdigest(), {})
def delete(self, location, connection=None):
location = location.store_location
if not connection:
connection = self.get_connection(location)
try:
# We request the manifest for the object. If one exists,
# that means the object was uploaded in chunks/segments,
# and we need to delete all the chunks as well as the
# manifest.
manifest = None
try:
headers = connection.head_object(location.container,
location.obj)
manifest = headers.get('x-object-manifest')
except swiftclient.ClientException as e:
if e.http_status != httplib.NOT_FOUND:
raise
if manifest:
# Delete all the chunks before the object manifest itself
obj_container, obj_prefix = manifest.split('/', 1)
segments = connection.get_container(obj_container,
prefix=obj_prefix)[1]
for segment in segments:
# TODO(jaypipes): This would be an easy area to parallelize
# since we're simply sending off parallelizable requests
# to Swift to delete stuff. It's not like we're going to
# be hogging up network or file I/O here...
connection.delete_object(obj_container, segment['name'])
# Delete object (or, in segmented case, the manifest)
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
def add(self, image_id, image_file, image_size, connection=None):
location = self.create_location(image_id)
if not connection:
connection = self.get_connection(location)
self._create_container_if_missing(location.container, connection)
LOG.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj, image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(int(
math.ceil(float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug(_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container, chunk_name, reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
with excutils.save_and_reraise_exception():
LOG.exception(_("Error during chunked upload to "
"backend, deleting stale chunks"))
self._delete_stale_chunks(connection,
location.container,
written_chunks)
bytes_read = reader.bytes_read
msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s") %
{'chunk_name': chunk_name,
'chunk_id': chunk_id,
'total_chunks': total_chunks,
'bytes_read': bytes_read,
'chunk_etag': chunk_etag})
LOG.debug(msg)
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug(_("Deleting final zero-length chunk"))
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container, location.obj,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag, {})
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
"this location"))
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % {'e': e})
LOG.error(msg)
raise glance.store.BackendException(msg)
def add(self, image_id, image_file):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
"""
swift_conn = self._make_swift_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, swift_conn, self.options)
obj_name = str(image_id)
location = StoreLocation({'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key})
logger.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
obj_etag = swift_conn.put_object(self.container, obj_name,
image_file)
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
# We do a HEAD on the newly-added image to determine the size
# of the image. A bit slow, but better than taking the word
# of the user adding the image with size attribute in the metadata
resp_headers = swift_conn.head_object(self.container, obj_name)
size = 0
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
size = int(resp_headers['content-length'])
return (location.get_uri(), size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
logger.error(msg)
raise glance.store.BackendException(msg)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:note Swift cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Swift and then write an manifest
to Swift that contains information about the chunks.
This same chunking process is used by default for images
of an unknown size, as pushing them directly to swift would
fail if the image turns out to be greater than 5GB.
"""
swift_conn = self._make_swift_connection(self.full_auth_address,
self.user,
self.key,
storage_url=self.storage_url,
token=self.token)
obj_name = str(image_id)
if self.multi_tenant:
# NOTE: When using multi-tenant we create containers for each
# image so we can set permissions on each image in swift
container = self.container + '_' + obj_name
auth_or_store_url = self.storage_url
else:
container = self.container
auth_or_store_url = self.auth_address
create_container_if_missing(container, swift_conn)
location = StoreLocation({
'scheme': self.scheme,
'container': container,
'obj': obj_name,
'auth_or_store_url': auth_or_store_url,
'user': self.user,
'key': self.key
})
LOG.debug(
_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = swift_conn.put_object(container,
obj_name,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug(
_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (obj_name, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
chunk_etag = swift_conn.put_object(
container,
chunk_name,
reader,
content_length=content_length)
bytes_read = reader.bytes_read
msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s")
LOG.debug(msg % locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if image_size == 0.
LOG.debug(_("Deleting final zero-length chunk"))
swift_conn.delete_object(container, chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the image_size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (container, obj_name)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
swift_conn.put_object(container,
obj_name,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swiftclient.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Swift already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
LOG.error(msg)
raise glance.store.BackendException(msg)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:note Swift cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Swift and then write an manifest
to Swift that contains information about the chunks.
This same chunking process is used by default for images
of an unknown size, as pushing them directly to swift would
fail if the image turns out to be greater than 5GB.
"""
swift_conn = self._make_swift_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, swift_conn, self.conf)
obj_name = str(image_id)
location = StoreLocation({'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key})
logger.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = swift_conn.put_object(self.container, obj_name,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(int(
math.ceil(float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
logger.debug(_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (obj_name, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
chunk_etag = swift_conn.put_object(
self.container, chunk_name, reader,
content_length=content_length)
bytes_read = reader.bytes_read
logger.debug(_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)s of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s")
% locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if image_size == 0.
logger.debug(_("Deleting final zero-length chunk"))
swift_conn.delete_object(self.container, chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the image_size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
swift_conn.put_object(self.container, obj_name,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
logger.error(msg)
raise glance.store.BackendException(msg)
def add(self, image_id, image_file, image_size, connection=None):
location = self.create_location(image_id)
if not connection:
connection = self.get_connection(location)
self._create_container_if_missing(location.container, connection)
LOG.debug(
_("Adding image object '%(obj_name)s' "
"to Swift") % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug(
_("Cannot determine image size. Adding as a "
"segmented object to Swift."))
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container,
chunk_name,
reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
self._delete_stale_chunks(connection,
location.container,
written_chunks)
raise
bytes_read = reader.bytes_read
msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s")
LOG.debug(msg % locals())
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug(_("Deleting final zero-length chunk"))
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container,
location.obj,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(
_("Swift already has an image at "
"this location"))
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
LOG.error(msg)
raise glance.store.BackendException(msg)
def add(cls, id, data, options):
"""
Stores image data to Swift and returns a location that the image was
written to.
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:param id: The opaque image identifier
:param data: The image data to write, as a file-like object
:param options: Conf mapping
:retval Tuple with (location, size)
The location that was written,
and the size in bytes of the data written
"""
from swift.common import client as swift_client
container = options.get('swift_store_container',
DEFAULT_SWIFT_CONTAINER)
# TODO(jaypipes): This needs to be checked every time
# because of the decision to make glance.store.Backend's
# interface all @classmethods. This is inefficient. Backend
# should be a stateful object with options parsed once in
# a constructor.
auth_address = cls._option_get(options, 'swift_store_auth_address')
user = cls._option_get(options, 'swift_store_user')
key = cls._option_get(options, 'swift_store_key')
scheme = 'swift+https'
if auth_address.startswith('http://'):
scheme = 'swift+http'
full_auth_address = auth_address
elif auth_address.startswith('https://'):
full_auth_address = auth_address
else:
full_auth_address = 'https://' + auth_address # Defaults https
swift_conn = swift_client.Connection(
authurl=full_auth_address, user=user, key=key, snet=False)
logger.debug("Adding image object to Swift using "
"(auth_address=%(auth_address)s, user=%(user)s, "
"key=%(key)s)" % locals())
create_container_if_missing(container, swift_conn, options)
obj_name = str(id)
location = StoreLocation({'scheme': scheme,
'container': container,
'obj': obj_name,
'authurl': auth_address,
'user': user,
'key': key})
try:
obj_etag = swift_conn.put_object(container, obj_name, data)
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
# We do a HEAD on the newly-added image to determine the size
# of the image. A bit slow, but better than taking the word
# of the user adding the image with size attribute in the metadata
resp_headers = swift_conn.head_object(container, obj_name)
size = 0
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
size = int(resp_headers['content-length'])
return (location.get_uri(), size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate("Swift already has an image at "
"location %(location)s" % locals())
msg = ("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s" % locals())
raise glance.store.BackendException(msg)
segments = connection.get_container(
obj_container, prefix=obj_prefix)[1]
for segment in segments:
# TODO(jaypipes): This would be an easy area to parallelize
# since we're simply sending off parallelizable requests
# to Swift to delete stuff. It's not like we're going to
# be hogging up network or file I/O here...
connection.delete_object(obj_container,
segment['name'])
# Delete object (or, in segmented case, the manifest)
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
msg = _("Swift could not find image at URI.")
raise exception.NotFound(msg)
else:
raise
def _create_container_if_missing(self, container, connection):
"""
Creates a missing container in Swift if the
``swift_store_create_container_on_put`` option is set.
:param container: Name of container to create
:param connection: Connection to swift service
"""
try:
connection.head_container(container)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:note Swift cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Swift and then write an manifest
to Swift that contains information about the chunks.
"""
swift_conn = self._make_swift_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, swift_conn, self.options)
obj_name = str(image_id)
location = StoreLocation({'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key})
logger.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
if image_size < self.large_object_size:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size, and we simply
# try to put the object into Swift and hope for the
# best...
obj_etag = swift_conn.put_object(self.container, obj_name,
image_file)
else:
# Write the image into Swift in chunks. We cannot
# stream chunks of the webob.Request.body_file, unfortunately,
# so we must write chunks of the body_file into a temporary
# disk buffer, and then pass this disk buffer to Swift.
bytes_left = image_size
chunk_id = 1
total_chunks = int(math.ceil(
float(image_size) / float(self.large_object_chunk_size)))
checksum = hashlib.md5()
while bytes_left > 0:
with tempfile.NamedTemporaryFile() as disk_buffer:
chunk_size = min(self.large_object_chunk_size,
bytes_left)
logger.debug(_("Writing %(chunk_size)d bytes for "
"chunk %(chunk_id)d/"
"%(total_chunks)d to disk buffer "
"for image %(image_id)s")
% locals())
chunk = image_file.read(chunk_size)
checksum.update(chunk)
disk_buffer.write(chunk)
disk_buffer.flush()
logger.debug(_("Writing chunk %(chunk_id)d/"
"%(total_chunks)d to Swift "
"for image %(image_id)s")
% locals())
chunk_etag = swift_conn.put_object(
self.container,
"%s-%05d" % (obj_name, chunk_id),
open(disk_buffer.name, 'rb'))
logger.debug(_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)d to Swift "
"returning MD5 of content: "
"%(chunk_etag)s")
% locals())
bytes_left -= self.large_object_chunk_size
chunk_id += 1
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
_ignored = swift_conn.put_object(self.container, obj_name,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
logger.error(msg)
raise glance.store.BackendException(msg)
def add(self, image_id, image_file, image_size):
"""
Stores an image file with supplied identifier to the backend
storage system and returns an `glance.store.ImageAddResult` object
containing information about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:retval `glance.store.ImageAddResult` object
:raises `glance.common.exception.Duplicate` if the image already
existed
Swift writes the image data using the scheme:
``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>`
where:
<USER> = ``swift_store_user``
<KEY> = ``swift_store_key``
<AUTH_ADDRESS> = ``swift_store_auth_address``
<CONTAINER> = ``swift_store_container``
<ID> = The id of the image being added
:note Swift auth URLs by default use HTTPS. To specify an HTTP
auth URL, you can specify http://someurl.com for the
swift_store_auth_address config option
:note Swift cannot natively/transparently handle objects >5GB
in size. So, if the image is greater than 5GB, we write
chunks of image data to Swift and then write an manifest
to Swift that contains information about the chunks.
"""
swift_conn = self._make_swift_connection(
auth_url=self.full_auth_address, user=self.user, key=self.key)
create_container_if_missing(self.container, swift_conn, self.options)
obj_name = str(image_id)
location = StoreLocation({'scheme': self.scheme,
'container': self.container,
'obj': obj_name,
'authurl': self.auth_address,
'user': self.user,
'key': self.key})
logger.debug(_("Adding image object '%(obj_name)s' "
"to Swift") % locals())
try:
if image_size < self.large_object_size:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size, and we simply
# try to put the object into Swift and hope for the
# best...
obj_etag = swift_conn.put_object(self.container, obj_name,
image_file)
if image_size == 0:
resp_headers = swift_conn.head_object(self.container,
obj_name)
# header keys are lowercased by Swift
if 'content-length' in resp_headers:
image_size = int(resp_headers['content-length'])
else:
# Write the image into Swift in chunks. We cannot
# stream chunks of the webob.Request.body_file, unfortunately,
# so we must write chunks of the body_file into a temporary
# disk buffer, and then pass this disk buffer to Swift.
bytes_left = image_size
chunk_id = 1
total_chunks = int(math.ceil(
float(image_size) / float(self.large_object_chunk_size)))
checksum = hashlib.md5()
while bytes_left > 0:
with tempfile.NamedTemporaryFile() as disk_buffer:
chunk_size = min(self.large_object_chunk_size,
bytes_left)
logger.debug(_("Writing %(chunk_size)d bytes for "
"chunk %(chunk_id)d/"
"%(total_chunks)d to disk buffer "
"for image %(image_id)s")
% locals())
chunk = image_file.read(chunk_size)
checksum.update(chunk)
disk_buffer.write(chunk)
disk_buffer.flush()
logger.debug(_("Writing chunk %(chunk_id)d/"
"%(total_chunks)d to Swift "
"for image %(image_id)s")
% locals())
chunk_etag = swift_conn.put_object(
self.container,
"%s-%05d" % (obj_name, chunk_id),
open(disk_buffer.name, 'rb'))
logger.debug(_("Wrote chunk %(chunk_id)d/"
"%(total_chunks)d to Swift "
"returning MD5 of content: "
"%(chunk_etag)s")
% locals())
bytes_left -= self.large_object_chunk_size
chunk_id += 1
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s" % (self.container, obj_name)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
_ignored = swift_conn.put_object(self.container, obj_name,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
return (location.get_uri(), image_size, obj_etag)
except swift_client.ClientException, e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
"location %s") % location.get_uri())
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %(e)s") % locals())
logger.error(msg)
raise glance.store.BackendException(msg)
segments = connection.get_container(
obj_container, prefix=obj_prefix)[1]
for segment in segments:
# TODO(jaypipes): This would be an easy area to parallelize
# since we're simply sending off parallelizable requests
# to Swift to delete stuff. It's not like we're going to
# be hogging up network or file I/O here...
connection.delete_object(
obj_container, segment['name'])
else:
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException, e:
if e.http_status == httplib.NOT_FOUND:
uri = location.get_uri()
raise exception.NotFound(_("Swift could not find image at "
"uri %(uri)s") % locals())
else:
raise
def _create_container_if_missing(self, container, connection):
"""
Creates a missing container in Swift if the
``swift_store_create_container_on_put`` option is set.
:param container: Name of container to create
:param connection: Connection to swift service
"""
try:
connection.head_container(container)
