def _create_container_if_missing(self, container, connection):
"""
Creates a missing container in Swift if the
``swift_store_create_container_on_put`` option is set.
:param container: Name of container to create
:param connection: Connection to swift service
"""
try:
connection.head_container(container)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
if self.conf.glance_store.swift_store_create_container_on_put:
try:
msg = (_LI("Creating swift container %(container)s") %
{
'container': container
})
LOG.info(msg)
connection.put_container(container)
except swiftclient.ClientException as e:
msg = (_("Failed to add container to Swift.\n"
"Got error from Swift: %s.") %
cutils.exception_to_str(e))
raise glance_store.BackendException(msg)
else:
msg = (_("The container %(container)s does not exist in "
"Swift. Please set the "
"swift_store_create_container_on_put option"
"to add container to Swift automatically.") % {
'container': container
})
raise glance_store.BackendException(msg)
else:
raise
def create_bucket_if_missing(conf, bucket, s3_conn):
"""
Creates a missing bucket in S3 if the
``s3_store_create_bucket_on_put`` option is set.
:param conf: Configuration
:param bucket: Name of bucket to create
:param s3_conn: Connection to S3
"""
from boto.exception import S3ResponseError
try:
s3_conn.get_bucket(bucket)
except S3ResponseError as e:
if e.status == http_client.NOT_FOUND:
if conf.glance_store.s3_store_create_bucket_on_put:
host = conf.glance_store.s3_store_host
location = get_s3_location(host)
try:
s3_conn.create_bucket(bucket, location=location)
except S3ResponseError as e:
msg = (_("Failed to add bucket to S3.\n"
"Got error from S3: %s.") %
utils.exception_to_str(e))
raise glance_store.BackendException(msg)
else:
msg = (_("The bucket %(bucket)s does not exist in "
"S3. Please set the "
"s3_store_create_bucket_on_put option "
"to add bucket to S3 automatically.") % {
'bucket': bucket
})
raise glance_store.BackendException(msg)
def _run_command(self, command, data, *params):
cmd = ['collie', 'vdi']
cmd.extend(command.split(' '))
cmd.extend(['-a', self.addr, '-p', self.port, self.name])
cmd.extend(params)
try:
return processutils.execute(*cmd, process_input=data)[0]
except processutils.ProcessExecutionError as exc:
LOG.error(exc)
raise glance_store.BackendException(exc)
def _run_command(self, command, data, *params):
cmd = ("collie vdi %(command)s -a %(addr)s -p %(port)d %(name)s "
"%(params)s" % {
"command": command,
"addr": self.addr,
"port": self.port,
"name": self.name,
"params": " ".join(map(str, params))
})
try:
return processutils.execute(cmd, process_input=data)[0]
except processutils.ProcessExecutionError as exc:
LOG.error(exc)
raise glance_store.BackendException(exc)
def add_multipart(self, image_file, image_size, bucket_obj, obj_name, loc):
"""
Stores an image file with a multi part upload to S3 backend
:param image_file: The image data to write, as a file-like object
:param bucket_obj: S3 bucket object
:param obj_name: The object name to be stored(image identifier)
:loc: The Store Location Info
"""
checksum = hashlib.md5()
pool_size = self.s3_store_thread_pools
pool = eventlet.greenpool.GreenPool(size=pool_size)
mpu = bucket_obj.initiate_multipart_upload(obj_name)
LOG.debug("Multipart initiate key=%(obj_name)s, "
"UploadId=%(UploadId)s" % {
'obj_name': obj_name,
'UploadId': mpu.id
})
cstart = 0
plist = []
chunk_size = int(math.ceil(float(image_size) / MAX_PART_NUM))
write_chunk_size = max(self.s3_store_large_object_chunk_size,
chunk_size)
it = utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE)
buffered_chunk = b''
while True:
try:
buffered_clen = len(buffered_chunk)
if buffered_clen < write_chunk_size:
# keep reading data
read_chunk = next(it)
buffered_chunk += read_chunk
continue
else:
write_chunk = buffered_chunk[:write_chunk_size]
remained_data = buffered_chunk[write_chunk_size:]
checksum.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, part)
plist.append(part)
cstart += 1
buffered_chunk = remained_data
except StopIteration:
if len(buffered_chunk) > 0:
# Write the last chunk data
write_chunk = buffered_chunk
checksum.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, part)
plist.append(part)
break
pedict = {}
total_size = 0
pool.waitall()
for part in plist:
pedict.update(part.etag)
total_size += part.size
success = True
for part in plist:
if not part.success:
success = False
if success:
# Complete
xml = get_mpu_xml(pedict)
bucket_obj.complete_multipart_upload(obj_name, mpu.id, xml)
checksum_hex = checksum.hexdigest()
LOG.info(
_LI("Multipart complete key=%(obj_name)s "
"UploadId=%(UploadId)s "
"Wrote %(total_size)d bytes to S3 key"
"named %(obj_name)s "
"with checksum %(checksum_hex)s") % {
'obj_name': obj_name,
'UploadId': mpu.id,
'total_size': total_size,
'checksum_hex': checksum_hex
})
return (loc.get_uri(), total_size, checksum_hex, {})
else:
# Abort
bucket_obj.cancel_multipart_upload(obj_name, mpu.id)
LOG.error(
_LE("Some parts failed to upload to S3. "
"Aborted the object key=%(obj_name)s") %
{'obj_name': obj_name})
msg = (_("Failed to add image object to S3. "
"key=%(obj_name)s") % {
'obj_name': obj_name
})
raise glance_store.BackendException(msg)
def add(self,
image_id,
image_file,
image_size,
connection=None,
context=None):
location = self.create_location(image_id, context=context)
if not connection:
connection = self.get_connection(location, context=context)
self._create_container_if_missing(location.container, connection)
LOG.debug("Adding image object '%(obj_name)s' "
"to Swift" % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj,
image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(
int(
math.ceil(
float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug("Cannot determine image size. Adding as a "
"segmented object to Swift.")
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container,
chunk_name,
reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
with excutils.save_and_reraise_exception():
LOG.exception(
_("Error during chunked upload to "
"backend, deleting stale chunks"))
self._delete_stale_chunks(connection,
location.container,
written_chunks)
bytes_read = reader.bytes_read
msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s" % {
'chunk_name': chunk_name,
'chunk_id': chunk_id,
'total_chunks': total_chunks,
'bytes_read': bytes_read,
'chunk_etag': chunk_etag
})
LOG.debug(msg)
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug("Deleting final zero-length chunk")
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {
'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest
}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container,
location.obj,
None,
headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
if sutils.is_multiple_swift_store_accounts_enabled(self.conf):
include_creds = False
else:
include_creds = True
return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, {})
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
msg = _("Swift already has an image at this location")
raise exceptions.Duplicate(message=msg)
msg = (_(u"Failed to add object to Swift.\n"
"Got error from Swift: %s.") % cutils.exception_to_str(e))
LOG.error(msg)
raise glance_store.BackendException(msg)
def _add_multipart(self, s3_client, image_file, image_size, bucket, key,
loc, hashing_algo, verifier):
"""Stores an image file with a multi part upload to S3 backend.
:param s3_client: An object with credentials to connect to S3
:param image_file: The image data to write, as a file-like object
:param bucket: S3 bucket name
:param key: The object name to be stored (image identifier)
:param loc: `glance_store.location.Location` object, supplied
from glance_store.location.get_location_from_uri()
:param hashing_algo: A hashlib algorithm identifier (string)
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
"""
os_hash_value = utils.get_hasher(hashing_algo, False)
checksum = utils.get_hasher('md5', False)
pool_size = self.s3_store_thread_pools
pool = eventlet.greenpool.GreenPool(size=pool_size)
mpu = s3_client.create_multipart_upload(Bucket=bucket, Key=key)
upload_id = mpu['UploadId']
LOG.debug("Multipart initiate key=%(key)s, UploadId=%(UploadId)s", {
'key': key,
'UploadId': upload_id
})
cstart = 0
plist = []
chunk_size = int(math.ceil(float(image_size) / MAX_PART_NUM))
write_chunk_size = max(self.s3_store_large_object_chunk_size,
chunk_size)
it = utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE)
buffered_chunk = b''
while True:
try:
buffered_clen = len(buffered_chunk)
if buffered_clen < write_chunk_size:
# keep reading data
read_chunk = next(it)
buffered_chunk += read_chunk
continue
else:
write_chunk = buffered_chunk[:write_chunk_size]
remained_data = buffered_chunk[write_chunk_size:]
os_hash_value.update(write_chunk)
checksum.update(write_chunk)
if verifier:
verifier.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part)
plist.append(part)
cstart += 1
buffered_chunk = remained_data
except StopIteration:
if len(buffered_chunk) > 0:
# Write the last chunk data
write_chunk = buffered_chunk
os_hash_value.update(write_chunk)
checksum.update(write_chunk)
if verifier:
verifier.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part)
plist.append(part)
break
pedict = {}
total_size = 0
pool.waitall()
for part in plist:
pedict.update(part.etag)
total_size += part.size
success = True
for part in plist:
if not part.success:
success = False
if success:
# Complete
mpu_list = self._get_mpu_list(pedict)
s3_client.complete_multipart_upload(Bucket=bucket,
Key=key,
MultipartUpload=mpu_list,
UploadId=upload_id)
hash_hex = os_hash_value.hexdigest()
checksum_hex = checksum.hexdigest()
# Add store backend information to location metadata
metadata = {}
if self.backend_group:
metadata['store'] = self.backend_group
LOG.info(
"Multipart complete key=%(key)s "
"UploadId=%(UploadId)s "
"Wrote %(total_size)d bytes to S3 key "
"named %(key)s "
"with checksum %(checksum)s", {
'key': key,
'UploadId': upload_id,
'total_size': total_size,
'checksum': checksum_hex
})
return loc.get_uri(), total_size, checksum_hex, hash_hex, metadata
# Abort
s3_client.abort_multipart_upload(Bucket=bucket,
Key=key,
UploadId=upload_id)
LOG.error("Some parts failed to upload to S3. "
"Aborted the key=%s", key)
msg = _("Failed to add image object to S3. key=%s") % key
raise glance_store.BackendException(msg)
def add(self,
image_id,
image_file,
image_size,
hashing_algo,
context=None,
verifier=None):
"""
Stores an image file with supplied identifier to the backend
storage system and returns a tuple containing information
about the stored image.
:param image_id: The opaque image identifier
:param image_file: The image data to write, as a file-like object
:param image_size: The size of the image data to write, in bytes
:param hashing_algo: A hashlib algorithm identifier (string)
:param context: A context object
:param verifier: An object used to verify signatures for images
:returns: tuple of: (1) URL in backing store, (2) bytes written,
(3) checksum, (4) multihash value, and (5) a dictionary
with storage system specific information
:raises: `glance_store.exceptions.Duplicate` if the image already
exists
"""
loc = StoreLocation(store_specs={
'scheme': self.scheme,
'bucket': self.bucket,
'key': image_id,
's3serviceurl': self.full_s3_host,
'accesskey': self.access_key,
'secretkey': self.secret_key
},
conf=self.conf,
backend_group=self.backend_group)
s3_client, bucket, key = self._operation_set(loc)
if not self._bucket_exists(s3_client, bucket):
if self._option_get('s3_store_create_bucket_on_put'):
self._create_bucket(s3_client,
self._option_get('s3_store_host'), bucket)
else:
msg = (_("The bucket %s does not exist in "
"S3. Please set the "
"s3_store_create_bucket_on_put option "
"to add bucket to S3 automatically.") % bucket)
raise glance_store.BackendException(msg)
LOG.debug(
"Adding image object to S3 using (s3_host=%(s3_host)s, "
"access_key=%(access_key)s, bucket=%(bucket)s, "
"key=%(key)s)", {
's3_host': self.s3_host,
'access_key': loc.accesskey,
'bucket': bucket,
'key': key
})
if not self._object_exists(s3_client, bucket, key):
if image_size < self.s3_store_large_object_size:
return self._add_singlepart(s3_client=s3_client,
image_file=image_file,
bucket=bucket,
key=key,
loc=loc,
hashing_algo=hashing_algo,
verifier=verifier)
return self._add_multipart(s3_client=s3_client,
image_file=image_file,
image_size=image_size,
bucket=bucket,
key=key,
loc=loc,
hashing_algo=hashing_algo,
verifier=verifier)
LOG.warning(
"S3 already has an image with bucket ID %(bucket)s, "
"key %(key)s", {
'bucket': bucket,
'key': key
})
raise exceptions.Duplicate(image=key)
def test_backend_exception(self):
msg = glance_store.BackendException()
self.assertIn(u'', encodeutils.exception_to_unicode(msg))
