def delete(self, location, context=None): """Takes a `glance_store.location.Location` object that indicates where to find the image file to delete :location `glance_store.location.Location` object, supplied from glance_store.location.get_location_from_uri() :raises NotFound if image does not exist """ file_path = '[%s] %s' % ( self.datastore_name, location.store_location.path[len(DS_URL_PREFIX):]) search_index_moref = self._service_content.searchIndex dc_moref = self._session.invoke_api(self._session.vim, 'FindByInventoryPath', search_index_moref, inventoryPath=self.datacenter_path) delete_task = self._session.invoke_api( self._session.vim, 'DeleteDatastoreFile_Task', self._service_content.fileManager, name=file_path, datacenter=dc_moref) try: self._session.wait_for_task(delete_task) except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_('Failed to delete image %(image)s content.') % {'image': location.image_id})
def delete(self, location, context=None): """Takes a `glance_store.location.Location` object that indicates where to find the image file to delete :location `glance_store.location.Location` object, supplied from glance_store.location.get_location_from_uri() :raises NotFound if image does not exist """ file_path = '[%s] %s' % (self.datastore_name, location.store_location. path[len(DS_URL_PREFIX):]) search_index_moref = self._service_content.searchIndex dc_moref = self._session.invoke_api(self._session.vim, 'FindByInventoryPath', search_index_moref, inventoryPath=self.datacenter_path) delete_task = self._session.invoke_api( self._session.vim, 'DeleteDatastoreFile_Task', self._service_content.fileManager, name=file_path, datacenter=dc_moref) try: self._session.wait_for_task(delete_task) except Exception: with excutils.save_and_reraise_exception(): LOG.exception( _LE('Failed to delete image %(image)s ' 'content.') % {'image': location.image_id})
def add(self, image_id, image_file, image_size, context=None): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exceptions.Duplicate` if the image already existed `glance.common.exceptions.UnexpectedStatus` if the upload request returned an unexpected status. The expected responses are 201 Created and 200 OK. """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation({ 'scheme': self.scheme, 'server_host': self.server_host, 'image_dir': self.store_image_dir, 'datacenter_path': self.datacenter_path, 'datastore_name': self.datastore_name, 'image_id': image_id }) cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = { 'Connection': 'Keep-Alive', 'Cookie': cookie, 'Transfer-Encoding': 'chunked' } try: conn = self._get_http_conn('PUT', loc, headers, content=image_file) res = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception( _('Failed to upload content of image ' '%(image)s') % {'image': image_id}) if res.status == httplib.CONFLICT: raise exceptions.Duplicate( _("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) if res.status not in (httplib.CREATED, httplib.OK): msg = (_('Failed to upload content of image %(image)s') % { 'image': image_id }) LOG.error(msg) raise exceptions.UnexpectedStatus(status=res.status, body=res.read()) return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def _query(self, location, method, depth=0): if depth > MAX_REDIRECTS: msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects.", { 'max_redirects': MAX_REDIRECTS }) LOG.debug(msg) raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) loc = location.store_location # NOTE(arnaud): use a decorator when the config is not tied to self for i in range(self.api_retry_count + 1): cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = {'Cookie': cookie} try: conn = self._get_http_conn(method, loc, headers) resp = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception( _LE('Failed to access image %(image)s ' 'content.') % {'image': location.image_id}) if resp.status >= 400: if resp.status == httplib.UNAUTHORIZED: self._create_session() continue if resp.status == httplib.NOT_FOUND: reason = _('VMware datastore could not find image at URI.') LOG.info(reason) raise exceptions.NotFound(message=reason) msg = ('HTTP request returned a %(status)s status code.' % { 'status': resp.status }) LOG.debug(msg) raise exceptions.BadStoreUri(msg) break location_header = resp.getheader('location') if location_header: if resp.status not in (301, 302): reason = (_("The HTTP URL %(path)s attempted to redirect " "with an invalid %(status)s status code.") % { 'path': loc.path, 'status': resp.status }) LOG.info(reason) raise exceptions.BadStoreUri(message=reason) location_class = glance_store.location.Location new_loc = location_class(location.store_name, location.store_location.__class__, uri=location_header, image_id=location.image_id, store_specs=location.store_specs) return self._query(new_loc, method, depth + 1) content_length = int(resp.getheader('content-length', 0)) return (conn, resp, content_length)
def add(self, image_id, image_file, image_size, context=None): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exceptions.Duplicate` if the image already existed `glance.common.exceptions.UnexpectedStatus` if the upload request returned an unexpected status. The expected responses are 201 Created and 200 OK. """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation( { "scheme": self.scheme, "server_host": self.server_host, "image_dir": self.store_image_dir, "datacenter_path": self.datacenter_path, "datastore_name": self.datastore_name, "image_id": image_id, } ) # NOTE(arnaud): use a decorator when the config is not tied to self for i in range(self.api_retry_count + 1): cookie = self._build_vim_cookie_header(self._session.vim.client.options.transport.cookiejar) headers = {"Connection": "Keep-Alive", "Cookie": cookie, "Transfer-Encoding": "chunked"} try: conn = self._get_http_conn("PUT", loc, headers, content=image_file) res = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE("Failed to upload content of image " "%(image)s"), {"image": image_id}) if res.status == httplib.UNAUTHORIZED: self._create_session() image_file.rewind() continue if res.status == httplib.CONFLICT: raise exceptions.Duplicate(_("Image file %(image_id)s already " "exists!") % {"image_id": image_id}) if res.status not in (httplib.CREATED, httplib.OK): msg = _LE("Failed to upload content of image %(image)s") % {"image": image_id} LOG.error(msg) raise exceptions.UnexpectedStatus(status=res.status, body=res.read()) break return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def remove_path_on_error(path, remove=delete_if_exists): """Protect code that wants to operate on PATH atomically. Any exception will cause PATH to be removed. :param path: File to work with :param remove: Optional function to remove passed path """ try: yield except Exception: with excutils.save_and_reraise_exception(): remove(path)
def add(self, image_id, image_file, image_size, context=None): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exceptions.Duplicate` if the image already existed `glance.common.exceptions.UnexpectedStatus` if the upload request returned an unexpected status. The expected responses are 201 Created and 200 OK. """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation({'scheme': self.scheme, 'server_host': self.server_host, 'image_dir': self.store_image_dir, 'datacenter_path': self.datacenter_path, 'datastore_name': self.datastore_name, 'image_id': image_id}) cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = {'Connection': 'Keep-Alive', 'Cookie': cookie, 'Transfer-Encoding': 'chunked'} try: conn = self._get_http_conn('PUT', loc, headers, content=image_file) res = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_('Failed to upload content of image ' '%(image)s') % {'image': image_id}) if res.status == httplib.CONFLICT: raise exceptions.Duplicate(_("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) if res.status not in (httplib.CREATED, httplib.OK): msg = (_('Failed to upload content of image %(image)s') % {'image': image_id}) LOG.error(msg) raise exceptions.UnexpectedStatus(status=res.status, body=res.read()) return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def _query(self, location, method, depth=0): if depth > MAX_REDIRECTS: msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects.", {"max_redirects": MAX_REDIRECTS}) LOG.debug(msg) raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) loc = location.store_location # NOTE(arnaud): use a decorator when the config is not tied to self for i in range(self.api_retry_count + 1): cookie = self._build_vim_cookie_header(self._session.vim.client.options.transport.cookiejar) headers = {"Cookie": cookie} try: conn = self._get_http_conn(method, loc, headers) resp = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE("Failed to access image %(image)s " "content.") % {"image": location.image_id}) if resp.status >= 400: if resp.status == httplib.UNAUTHORIZED: self._create_session() continue if resp.status == httplib.NOT_FOUND: reason = _("VMware datastore could not find image at URI.") LOG.info(reason) raise exceptions.NotFound(message=reason) msg = "HTTP request returned a %(status)s status code." % {"status": resp.status} LOG.debug(msg) raise exceptions.BadStoreUri(msg) break location_header = resp.getheader("location") if location_header: if resp.status not in (301, 302): reason = _("The HTTP URL %(path)s attempted to redirect " "with an invalid %(status)s status code.") % { "path": loc.path, "status": resp.status, } LOG.info(reason) raise exceptions.BadStoreUri(message=reason) location_class = glance_store.location.Location new_loc = location_class( location.store_name, location.store_location.__class__, uri=location_header, image_id=location.image_id, store_specs=location.store_specs, ) return self._query(new_loc, method, depth + 1) content_length = int(resp.getheader("content-length", 0)) return (conn, resp, content_length)
def _query(self, location, method, headers, depth=0): if depth > MAX_REDIRECTS: msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects." % { 'max_redirects': MAX_REDIRECTS }) LOG.debug(msg) raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) loc = location.store_location try: conn = self._get_http_conn(method, loc, headers) resp = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception( _('Failed to access image %(image)s content.') % {'image': location.image_id}) if resp.status >= 400: if resp.status == httplib.NOT_FOUND: reason = _('VMware datastore could not find image at URI.') LOG.info(reason) raise exceptions.NotFound(message=reason) msg = ('HTTP request returned a %(status)s status code.' % { 'status': resp.status }) LOG.debug(msg) raise exceptions.BadStoreUri(msg) location_header = resp.getheader('location') if location_header: if resp.status not in (301, 302): reason = (_("The HTTP URL %(path)s attempted to redirect " "with an invalid %(status)s status code.") % { 'path': loc.path, 'status': resp.status }) LOG.info(reason) raise exceptions.BadStoreUri(message=reason) location_class = glance_store.location.Location new_loc = location_class(location.store_name, location.store_location.__class__, uri=location_header, image_id=location.image_id, store_specs=location.store_specs) return self._query(new_loc, method, depth + 1) content_length = int(resp.getheader('content-length', 0)) return (conn, resp, content_length)
def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance_store.exceptions.Duplicate` if the image already existed """ loc = StoreLocation({'image_id': image_id}) if self.fs.exists(image_id): raise exceptions.Duplicate( _("GridFS already has an image at " "location %s") % loc.get_uri()) LOG.debug( _("Adding a new image to GridFS with " "id %(iid)s and size %(size)s") % dict(iid=image_id, size=image_size)) try: self.fs.put(image_file, _id=image_id) image = self._get_file(loc) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self.fs.delete(image_id) LOG.debug( _("Uploaded image %(iid)s, " "md5 %(md)s, length %(len)s to GridFS") % dict(iid=image._id, md=image.md5, len=image.length)) return (loc.get_uri(), image.length, image.md5, {})
def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, and checksum :raises `glance_store.exceptions.Duplicate` if the image already existed """ image = SheepdogImage(self.addr, self.port, image_id, self.WRITE_CHUNKSIZE) if image.exist(): raise exceptions.Duplicate( _("Sheepdog image %s already exists") % image_id) location = StoreLocation({'image': image_id}) checksum = hashlib.md5() image.create(image_size) try: total = left = image_size while left > 0: length = min(self.chunk_size, left) data = image_file.read(length) image.write(data, total - left, length) left -= length checksum.update(data) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exceptions. with excutils.save_and_reraise_exception(): image.delete() return (location.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, and checksum :raises `glance_store.exceptions.Duplicate` if the image already existed """ image = SheepdogImage(self.addr, self.port, image_id, self.WRITE_CHUNKSIZE) if image.exist(): raise exceptions.Duplicate(_("Sheepdog image %s already exists") % image_id) location = StoreLocation({'image': image_id}) checksum = hashlib.md5() image.create(image_size) try: total = left = image_size while left > 0: length = min(self.chunk_size, left) data = image_file.read(length) image.write(data, total - left, length) left -= length checksum.update(data) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exceptions. with excutils.save_and_reraise_exception(): image.delete() return (location.get_uri(), image_size, checksum.hexdigest(), {})
def _query(self, location, method, headers, depth=0): if depth > MAX_REDIRECTS: msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects." % {'max_redirects': MAX_REDIRECTS}) LOG.debug(msg) raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) loc = location.store_location try: conn = self._get_http_conn(method, loc, headers) resp = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_('Failed to access image %(image)s content.') % {'image': location.image_id}) if resp.status >= 400: if resp.status == httplib.NOT_FOUND: reason = _('VMware datastore could not find image at URI.') LOG.info(reason) raise exceptions.NotFound(message=reason) msg = ('HTTP request returned a %(status)s status code.' % {'status': resp.status}) LOG.debug(msg) raise exceptions.BadStoreUri(msg) location_header = resp.getheader('location') if location_header: if resp.status not in (301, 302): reason = (_("The HTTP URL %(path)s attempted to redirect " "with an invalid %(status)s status code.") % {'path': loc.path, 'status': resp.status}) LOG.info(reason) raise exceptions.BadStoreUri(message=reason) location_class = glance_store.location.Location new_loc = location_class(location.store_name, location.store_location.__class__, uri=location_header, image_id=location.image_id, store_specs=location.store_specs) return self._query(new_loc, method, depth + 1) content_length = int(resp.getheader('content-length', 0)) return (conn, resp, content_length)
def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance_store.exceptions.Duplicate` if the image already existed """ loc = StoreLocation({'image_id': image_id}) if self.fs.exists(image_id): raise exceptions.Duplicate(_("GridFS already has an image at " "location %s") % loc.get_uri()) LOG.debug(_("Adding a new image to GridFS with " "id %(iid)s and size %(size)s") % dict(iid=image_id, size=image_size)) try: self.fs.put(image_file, _id=image_id) image = self._get_file(loc) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self.fs.delete(image_id) LOG.debug(_("Uploaded image %(iid)s, " "md5 %(md)s, length %(len)s to GridFS") % dict(iid=image._id, md=image.md5, len=image.length)) return (loc.get_uri(), image.length, image.md5, {})
def add(self, image_id, image_file, image_size, connection=None, context=None): location = self.create_location(image_id, context=context) if not connection: connection = self.get_connection(location, context=context) self._create_container_if_missing(location.container, connection) LOG.debug("Adding image object '%(obj_name)s' " "to Swift" % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str( int( math.ceil( float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug("Cannot determine image size. Adding as a " "segmented object to Swift.") total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend with excutils.save_and_reraise_exception(): LOG.exception( _("Error during chunked upload to " "backend, deleting stale chunks")) self._delete_stale_chunks(connection, location.container, written_chunks) bytes_read = reader.bytes_read msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s" % { 'chunk_name': chunk_name, 'chunk_id': chunk_id, 'total_chunks': total_chunks, 'bytes_read': bytes_read, 'chunk_etag': chunk_etag }) LOG.debug(msg) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if size == 0. LOG.debug("Deleting final zero-length chunk") connection.delete_object(location.container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = { 'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest } # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details if sutils.is_multiple_swift_store_accounts_enabled(): include_creds = False else: include_creds = True return (location.get_uri(credentials_included=include_creds), image_size, obj_etag, {}) except swiftclient.ClientException as e: if e.http_status == httplib.CONFLICT: msg = _("Swift already has an image at this location") raise exceptions.Duplicate(message=msg) msg = (_(u"Failed to add object to Swift.\n" "Got error from Swift: %s") % unicode(e)) LOG.error(msg) raise glance_store.BackendException(msg)
def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance_store.exceptions.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exceptions.Duplicate(image=filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) errors = { errno.EFBIG: exceptions.StorageFull(), errno.ENOSPC: exceptions.StorageFull(), errno.EACCES: exceptions.StorageWriteDenied() } raise errors.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), { 'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex }) if self.conf.glance_store.filesystem_store_file_perm > 0: perm = int(str(self.conf.glance_store.filesystem_store_file_perm), 8) try: os.chmod(filepath, perm) except (IOError, OSError): LOG.warn( _LW("Unable to set permission to image: %s") % filepath) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def rewind(self): try: self.data.seek(0) except IOError: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to rewind image content'))
def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance_store.exceptions.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exceptions.Duplicate(image=filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) errors = {errno.EFBIG: exceptions.StorageFull(), errno.ENOSPC: exceptions.StorageFull(), errno.EACCES: exceptions.StorageWriteDenied()} raise errors.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug(_("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), {'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex}) if self.conf.glance_store.filesystem_store_file_perm > 0: perm = int(str(self.conf.glance_store.filesystem_store_file_perm), 8) try: os.chmod(filepath, perm) except (IOError, OSError): LOG.warn(_LW("Unable to set permission to image: %s") % filepath) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size, connection=None, context=None): location = self.create_location(image_id, context=context) if not connection: connection = self.get_connection(location, context=context) self._create_container_if_missing(location.container, connection) LOG.debug("Adding image object '%(obj_name)s' " "to Swift" % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str(int( math.ceil(float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug("Cannot determine image size. Adding as a " "segmented object to Swift.") total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend with excutils.save_and_reraise_exception(): LOG.exception(_("Error during chunked upload to " "backend, deleting stale chunks")) self._delete_stale_chunks(connection, location.container, written_chunks) bytes_read = reader.bytes_read msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s" % {'chunk_name': chunk_name, 'chunk_id': chunk_id, 'total_chunks': total_chunks, 'bytes_read': bytes_read, 'chunk_etag': chunk_etag}) LOG.debug(msg) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if size == 0. LOG.debug("Deleting final zero-length chunk") connection.delete_object(location.container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = {'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest} # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details if sutils.is_multiple_swift_store_accounts_enabled(): include_creds = False else: include_creds = True return (location.get_uri(credentials_included=include_creds), image_size, obj_etag, {}) except swiftclient.ClientException as e: if e.http_status == httplib.CONFLICT: msg = _("Swift already has an image at this location") raise exceptions.Duplicate(message=msg) msg = (_(u"Failed to add object to Swift.\n" "Got error from Swift: %s") % unicode(e)) LOG.error(msg) raise glance_store.BackendException(msg)
def rewind(self): try: self.data.seek(0) except IOError: with excutils.save_and_reraise_exception(): LOG.exception(_LE("Failed to rewind image content"))