def add(self, image_id, image_file, image_size, connection=None, context=None, verifier=None): location = self.create_location(image_id, context=context) if not connection: connection = self.get_connection(location, context=context) self._create_container_if_missing(location.container, connection) LOG.debug("Adding image object '%(obj_name)s' " "to Swift" % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str(int( math.ceil(float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug("Cannot determine image size. Adding as a " "segmented object to Swift.") total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size, verifier) if reader.is_zero_size is True: LOG.debug('Not writing zero-length chunk.') break try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend with excutils.save_and_reraise_exception(): LOG.exception(_("Error during chunked upload to " "backend, deleting stale chunks")) self._delete_stale_chunks(connection, location.container, written_chunks) bytes_read = reader.bytes_read msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s" % {'chunk_name': chunk_name, 'chunk_id': chunk_id, 'total_chunks': total_chunks, 'bytes_read': bytes_read, 'chunk_etag': chunk_etag}) LOG.debug(msg) chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = {'ETag': hashlib.md5(b"").hexdigest(), 'X-Object-Manifest': manifest} # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details if sutils.is_multiple_swift_store_accounts_enabled(self.conf): include_creds = False else: include_creds = True return (location.get_uri(credentials_included=include_creds), image_size, obj_etag, {}) except swiftclient.ClientException as e: if e.http_status == http_client.CONFLICT: msg = _("Swift already has an image at this location") raise exceptions.Duplicate(message=msg) msg = (_(u"Failed to add object to Swift.\n" "Got error from Swift: %s.") % encodeutils.exception_to_unicode(e)) LOG.error(msg) raise glance_store.BackendException(msg)
def test_multiple_swift_account_enabled(self): self.config(swift_store_config_file="glance-swift.conf") self.assertTrue( sutils.is_multiple_swift_store_accounts_enabled(self.conf))
def test_multiple_swift_account_disabled(self): self.config(swift_store_config_file=None) self.assertFalse( sutils.is_multiple_swift_store_accounts_enabled(self.conf))
def add(self, image_id, image_file, image_size, connection=None, context=None): location = self.create_location(image_id, context=context) if not connection: connection = self.get_connection(location, context=context) self._create_container_if_missing(location.container, connection) LOG.debug("Adding image object '%(obj_name)s' " "to Swift" % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str( int( math.ceil( float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug("Cannot determine image size. Adding as a " "segmented object to Swift.") total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend with excutils.save_and_reraise_exception(): LOG.exception( _("Error during chunked upload to " "backend, deleting stale chunks")) self._delete_stale_chunks(connection, location.container, written_chunks) bytes_read = reader.bytes_read msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s" % { 'chunk_name': chunk_name, 'chunk_id': chunk_id, 'total_chunks': total_chunks, 'bytes_read': bytes_read, 'chunk_etag': chunk_etag }) LOG.debug(msg) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if size == 0. LOG.debug("Deleting final zero-length chunk") connection.delete_object(location.container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = { 'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest } # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details if sutils.is_multiple_swift_store_accounts_enabled(self.conf): include_creds = False else: include_creds = True return (location.get_uri(credentials_included=include_creds), image_size, obj_etag, {}) except swiftclient.ClientException as e: if e.http_status == httplib.CONFLICT: msg = _("Swift already has an image at this location") raise exceptions.Duplicate(message=msg) msg = (_(u"Failed to add object to Swift.\n" "Got error from Swift: %s.") % cutils.exception_to_str(e)) LOG.error(msg) raise glance_store.BackendException(msg)