def _delete_image(self, target_pool, image_name, snapshot_name=None, context=None): """ Delete RBD image and snapshot. :param image_name: Image's name :param snapshot_name: Image snapshot's name :raises: NotFound if image does not exist; InUseByStore if image is in use or snapshot unprotect failed """ with self.get_connection(conffile=self.conf_file, rados_id=self.user) as conn: with conn.open_ioctx(target_pool) as ioctx: try: # First remove snapshot. if snapshot_name is not None: with rbd.Image(ioctx, image_name) as image: try: self._unprotect_snapshot(image, snapshot_name) image.remove_snap(snapshot_name) except rbd.ImageNotFound as exc: msg = (_("Snap Operating Exception " "%(snap_exc)s " "Snapshot does not exist.") % { 'snap_exc': exc }) LOG.debug(msg) except rbd.ImageBusy as exc: log_msg = (_LW("Snap Operating Exception " "%(snap_exc)s " "Snapshot is in use.") % { 'snap_exc': exc }) LOG.warning(log_msg) raise exceptions.InUseByStore() # Then delete image. rbd.RBD().remove(ioctx, image_name) except rbd.ImageHasSnapshots: log_msg = (_LW("Remove image %(img_name)s failed. " "It has snapshot(s) left.") % { 'img_name': image_name }) LOG.warning(log_msg) raise exceptions.HasSnapshot() except rbd.ImageBusy: log_msg = (_LW("Remove image %(img_name)s failed. " "It is in use.") % { 'img_name': image_name }) LOG.warning(log_msg) raise exceptions.InUseByStore() except rbd.ImageNotFound: msg = _("RBD image %s does not exist") % image_name raise exceptions.NotFound(message=msg)
def _set_exec_permission(self, datadir): """ Set the execution permission of owner-group and/or other-users to image directory if the image file which contained needs relevant access permissions. :datadir is a directory path in which glance writes image files. """ if self.backend_group: fstore_perm = getattr( self.conf, self.backend_group).filesystem_store_file_perm else: fstore_perm = self.conf.glance_store.filesystem_store_file_perm if fstore_perm <= 0: return try: mode = os.stat(datadir)[stat.ST_MODE] perm = int(str(fstore_perm), 8) if perm & stat.S_IRWXO > 0: if not mode & stat.S_IXOTH: # chmod o+x mode |= stat.S_IXOTH os.chmod(datadir, mode) if perm & stat.S_IRWXG > 0: if not mode & stat.S_IXGRP: # chmod g+x os.chmod(datadir, mode | stat.S_IXGRP) except (IOError, OSError): LOG.warning(_LW("Unable to set execution permission of " "owner-group and/or other-users to datadir: %s") % datadir)
def op_checker(store, *args, **kwargs): get_capabilities = [ BitMasks.READ_ACCESS, BitMasks.READ_OFFSET if kwargs.get('offset') else BitMasks.NONE, BitMasks.READ_CHUNK if kwargs.get('chunk_size') else BitMasks.NONE ] op_cap_map = { 'get': get_capabilities, 'add': [BitMasks.WRITE_ACCESS], 'delete': [BitMasks.WRITE_ACCESS]} op_exec_map = { 'get': (exceptions.StoreRandomGetNotSupported if kwargs.get('offset') or kwargs.get('chunk_size') else exceptions.StoreGetNotSupported), 'add': exceptions.StoreAddDisabled, 'delete': exceptions.StoreDeleteNotSupported} op = store_op_fun.__name__.lower() try: req_cap = op_cap_map[op] except KeyError: LOG.warning(_LW('The capability of operation "%s" ' 'could not be checked.'), op) else: if not store.is_capable(*req_cap): kwargs.setdefault('offset', 0) kwargs.setdefault('chunk_size', None) raise op_exec_map[op](**kwargs) return store_op_fun(store, *args, **kwargs)
def _set_exec_permission(self, datadir): """ Set the execution permission of owner-group and/or other-users to image directory if the image file which contained needs relevant access permissions. :datadir is a directory path in which glance writes image files. """ if self.conf.glance_store.filesystem_store_file_perm <= 0: return try: mode = os.stat(datadir)[stat.ST_MODE] perm = int(str(self.conf.glance_store.filesystem_store_file_perm), 8) if perm & stat.S_IRWXO > 0: if not mode & stat.S_IXOTH: # chmod o+x mode |= stat.S_IXOTH os.chmod(datadir, mode) if perm & stat.S_IRWXG > 0: if not mode & stat.S_IXGRP: # chmod g+x os.chmod(datadir, mode | stat.S_IXGRP) except (IOError, OSError): LOG.warning(_LW("Unable to set execution permission of " "owner-group and/or other-users to datadir: %s") % datadir)
def configure_add(self): """ Check to verify if the volume types configured for the cinder store exist in deployment and if not, log a warning. """ cinder_volume_type = self.store_conf.cinder_volume_type if cinder_volume_type: # NOTE: `cinder_volume_type` is configured, check # configured volume_type is available in cinder or not cinder_client = self.get_cinderclient() try: # We don't even need the volume type object, as long # as this returns clean, we know the name is good. cinder_client.volume_types.find(name=cinder_volume_type) # No need to worry about a NoUniqueMatch as volume type name # is unique except cinder_exception.NotFound: reason = (_LW("Invalid `cinder_volume_type %s`" % cinder_volume_type)) LOG.warning(reason) except cinder_exception.ClientException: pass
def op_checker(store, *args, **kwargs): # NOTE(zhiyan): Trigger the hook of updating store # dynamic capabilities based on current store status. if store.conf.glance_store.store_capabilities_update_min_interval > 0: _schedule_capabilities_update(store) get_capabilities = [ BitMasks.READ_ACCESS, BitMasks.READ_OFFSET if kwargs.get('offset') else BitMasks.NONE, BitMasks.READ_CHUNK if kwargs.get('chunk_size') else BitMasks.NONE ] op_cap_map = { 'get': get_capabilities, 'add': [BitMasks.WRITE_ACCESS], 'delete': [BitMasks.WRITE_ACCESS]} op_exec_map = { 'get': (exceptions.StoreRandomGetNotSupported if kwargs.get('offset') or kwargs.get('chunk_size') else exceptions.StoreGetNotSupported), 'add': exceptions.StoreAddDisabled, 'delete': exceptions.StoreDeleteNotSupported} op = store_op_fun.__name__.lower() try: req_cap = op_cap_map[op] except KeyError: LOG.warning(_LW('The capability of operation "%s" ' 'could not be checked.'), op) else: if not store.is_capable(*req_cap): kwargs.setdefault('offset', 0) kwargs.setdefault('chunk_size', None) raise op_exec_map[op](**kwargs) return store_op_fun(store, *args, **kwargs)
def umount(self, vol_name, mountpoint, host, rootwrap_helper): """Mark an attachment as no longer in use, and unmount its mountpoint if necessary. :param vol_name: The name of the volume on the remote filesystem. :param mountpoint: The directory where the filesystem is be mounted on the local compute host. :param host: The host the volume was attached to. """ LOG.debug('_HostMountState.umount(vol_name=%(vol_name)s, ' 'mountpoint=%(mountpoint)s)', {'vol_name': vol_name, 'mountpoint': mountpoint}) with self._get_locked(mountpoint) as mount: try: mount.remove_attachment(vol_name, host) except KeyError: LOG.warning(_LW("Request to remove attachment " "(%(vol_name)s, %(host)s) from " "%(mountpoint)s, but we don't think it's in " "use."), {'vol_name': vol_name, 'host': host, 'mountpoint': mountpoint}) if not mount.in_use(): mounted = os.path.ismount(mountpoint) if mounted: mounted = self._real_umount(mountpoint, rootwrap_helper) # Delete our record entirely if it's unmounted if not mounted: del self.mountpoints[mountpoint] LOG.debug('_HostMountState.umount() for %(mountpoint)s ' 'completed successfully', {'mountpoint': mountpoint})
def add(self, image_id, image_file, image_size, hashing_algo, context=None, verifier=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :param hashing_algo: A hashlib algorithm identifier (string) :param context: The request context :param verifier: An object used to verify signatures for images :returns: tuple of: (1) URL in backing store, (2) bytes written, (3) checksum, (4) multihash value, and (5) a dictionary with storage system specific information :raises: `glance_store.exceptions.Duplicate` if the image already exists :note:: By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exceptions.Duplicate(image=filepath) os_hash_value = hashlib.new(str(hashing_algo)) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): bytes_written += len(buf) os_hash_value.update(buf) checksum.update(buf) if verifier: verifier.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) errors = {errno.EFBIG: exceptions.StorageFull(), errno.ENOSPC: exceptions.StorageFull(), errno.EACCES: exceptions.StorageWriteDenied()} raise errors.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) hash_hex = os_hash_value.hexdigest() checksum_hex = checksum.hexdigest() metadata = self._get_metadata(filepath) LOG.debug(("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s and multihash %(hash_hex)s"), {'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex, 'hash_hex': hash_hex}) if self.backend_group: fstore_perm = getattr( self.conf, self.backend_group).filesystem_store_file_perm else: fstore_perm = self.conf.glance_store.filesystem_store_file_perm if fstore_perm > 0: perm = int(str(fstore_perm), 8) try: os.chmod(filepath, perm) except (IOError, OSError): LOG.warning(_LW("Unable to set permission to image: %s") % filepath) # Add store backend information to location metadata if self.backend_group: metadata['backend'] = u"%s" % self.backend_group return ('file://%s' % filepath, bytes_written, checksum_hex, hash_hex, metadata)
def __init__(self, *args, **kargs): super(Store, self).__init__(*args, **kargs) LOG.warning( _LW("Cinder store is considered experimental. " "Current deployers should be aware that the use " "of it in production right now may be risky."))
def add(self, image_id, image_file, image_size, context=None, verifier=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :param verifier: An object used to verify signatures for images :retval: tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises: `glance_store.exceptions.Duplicate` if the image already existed :note:: By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ #拼接出存放路径 datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) #文件已存在,报错 if os.path.exists(filepath): raise exceptions.Duplicate(image=filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): bytes_written += len(buf) #计算checksum checksum.update(buf) if verifier: verifier.update(buf) #写入文件 f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) errors = { errno.EFBIG: exceptions.StorageFull(), errno.ENOSPC: exceptions.StorageFull(), errno.EACCES: exceptions.StorageWriteDenied() } raise errors.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata(filepath) LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), { 'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex }) if self.conf.glance_store.filesystem_store_file_perm > 0: perm = int(str(self.conf.glance_store.filesystem_store_file_perm), 8) try: #修改权限位 os.chmod(filepath, perm) except (IOError, OSError): LOG.warning( _LW("Unable to set permission to image: %s") % filepath) #返回数据写结果 return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size, hashing_algo, context=None, verifier=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :param hashing_algo: A hashlib algorithm identifier (string) :param context: A context object :param verifier: An object used to verify signatures for images :returns: tuple of: (1) URL in backing store, (2) bytes written, (3) checksum, (4) multihash value, and (5) a dictionary with storage system specific information :raises: `glance_store.exceptions.Duplicate` if the image already exists """ os_hash_value = utils.get_hasher(hashing_algo, False) checksum = utils.get_hasher('md5', False) image_name = str(image_id) with self.get_connection(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): # Librados's get_fsid is represented as binary # in py3 instead of str as it is in py2. # This is causing problems with ceph. # Decode binary to str fixes these issues. # Fix with encodeutils.safe_decode CAN BE REMOVED # after librados's fix will be stable. # # More informations: # https://bugs.launchpad.net/glance-store/+bug/1816721 # https://bugs.launchpad.net/cinder/+bug/1816468 # https://tracker.ceph.com/issues/38381 fsid = encodeutils.safe_decode(conn.get_fsid()) with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.WRITE_CHUNKSIZE, 2)) LOG.debug('creating image %s with order %d and size %d', image_name, order, image_size) if image_size == 0: LOG.warning( _LW("Since image size is zero we will be " "doing resize-before-write which will be " "slower than normal")) try: loc = self._create_image(fsid, conn, ioctx, image_name, image_size, order) except rbd.ImageExists: msg = _('RBD image %s already exists') % image_id raise exceptions.Duplicate(message=msg) try: with rbd.Image(ioctx, image_name) as image: bytes_written = 0 offset = 0 chunks = utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE) for chunk in chunks: # NOTE(jokke): If we don't know image size we need # to resize it on write. The resize amount will # ramp up to 8 gigs. chunk_length = len(chunk) self.size = self._resize_on_write( image, image_size, bytes_written, chunk_length) bytes_written += chunk_length if not (self.thin_provisioning and not any(chunk)): image.write(chunk, offset) offset += chunk_length os_hash_value.update(chunk) checksum.update(chunk) if verifier: verifier.update(chunk) # Lets trim the image in case we overshoot with resize if image_size == 0: image.resize(bytes_written) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except rbd.NoSpace: log_msg = (_LE("Failed to store image %(img_name)s " "insufficient space available") % { 'img_name': image_name }) LOG.error(log_msg) # Delete image if one was created try: target_pool = loc.pool or self.pool self._delete_image(target_pool, loc.image, loc.snapshot) except exceptions.NotFound: pass raise exceptions.StorageFull(message=log_msg) except Exception as exc: log_msg = (_LE("Failed to store image %(img_name)s " "Store Exception %(store_exc)s") % { 'img_name': image_name, 'store_exc': exc }) LOG.error(log_msg) # Delete image if one was created try: target_pool = loc.pool or self.pool self._delete_image(target_pool, loc.image, loc.snapshot) except exceptions.NotFound: pass raise exc # Make sure we send back the image size whether provided or inferred. if image_size == 0: image_size = bytes_written # Add store backend information to location metadata metadata = {} if self.backend_group: metadata['store'] = u"%s" % self.backend_group return (loc.get_uri(), image_size, checksum.hexdigest(), os_hash_value.hexdigest(), metadata)
def __init__(self, *args, **kargs): super(Store, self).__init__(*args, **kargs) LOG.warning(_LW("Cinder store is considered experimental. " "Current deployers should be aware that the use " "of it in production right now may be risky."))
def add(self, image_id, image_file, image_size, context=None, verifier=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :param verifier: An object used to verify signatures for images :retval: tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises: `glance_store.exceptions.Duplicate` if the image already existed :note:: By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exceptions.Duplicate(image=filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) if verifier: verifier.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) errors = {errno.EFBIG: exceptions.StorageFull(), errno.ENOSPC: exceptions.StorageFull(), errno.EACCES: exceptions.StorageWriteDenied()} raise errors.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata(filepath) LOG.debug(_("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), {'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex}) if self.conf.glance_store.filesystem_store_file_perm > 0: perm = int(str(self.conf.glance_store.filesystem_store_file_perm), 8) try: os.chmod(filepath, perm) except (IOError, OSError): LOG.warning(_LW("Unable to set permission to image: %s") % filepath) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)