def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ LOG.debug("Snapshot: %(snap)s, volume: %(vol)s, " "volume_size: %(size)s" % {'snap': snapshot['id'], 'vol': volume['id'], 'size': volume_size}) info_path = self._local_path_volume_info(snapshot['volume']) snap_info = self._read_info_file(info_path) vol_dir = self._local_volume_dir(snapshot['volume']) out_format = self.get_volume_format(volume, qemu_format=True) forward_file = snap_info[snapshot['id']] forward_path = os.path.join(vol_dir, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot['volume']['name']) path_to_snap_img = os.path.join(vol_dir, img_info.backing_file) LOG.debug("Will copy from snapshot at %s" % path_to_snap_img) image_utils.convert_image(path_to_snap_img, self.local_path(volume), out_format) self._extend_volume(volume, volume_size) self._set_rw_permissions_for_all(self.local_path(volume))
def test_clone_image_cloneableshare_notraw(self): drv = self._driver mox = self.mox volume = {"name": "vol", "size": "20"} mox.StubOutWithMock(drv, "_find_image_in_cache") mox.StubOutWithMock(drv, "_is_cloneable_share") mox.StubOutWithMock(drv, "_get_mount_point_for_share") mox.StubOutWithMock(image_utils, "qemu_img_info") mox.StubOutWithMock(drv, "_clone_volume") mox.StubOutWithMock(drv, "_discover_file_till_timeout") mox.StubOutWithMock(drv, "_set_rw_permissions_for_all") mox.StubOutWithMock(drv, "_resize_image_file") mox.StubOutWithMock(image_utils, "convert_image") mox.StubOutWithMock(drv, "_register_image_in_cache") mox.StubOutWithMock(drv, "_is_share_vol_compatible") drv._find_image_in_cache(IgnoreArg()).AndReturn([]) drv._is_cloneable_share("nfs://127.0.0.1/share/img-id").AndReturn("127.0.0.1:/share") drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True) drv._get_mount_point_for_share("127.0.0.1:/share").AndReturn("/mnt") image_utils.qemu_img_info("/mnt/img-id").AndReturn(self.get_img_info("notraw")) image_utils.convert_image(IgnoreArg(), IgnoreArg(), "raw") image_utils.qemu_img_info("/mnt/vol").AndReturn(self.get_img_info("raw")) drv._register_image_in_cache(IgnoreArg(), IgnoreArg()) drv._get_mount_point_for_share("127.0.0.1:/share").AndReturn("/mnt") drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True) drv._set_rw_permissions_for_all("/mnt/vol") drv._resize_image_file({"name": "vol"}, IgnoreArg()) mox.ReplayAll() drv.clone_image(volume, ("nfs://127.0.0.1/share/img-id", None), "image_id") mox.VerifyAll()
def copy_volume_to_image(self, context, volume, image_service, image_meta): """Copy the volume to the specified image.""" # If snapshots exist, flatten to a temporary image, and upload it active_file = self.get_active_image_from_info(volume) active_file_path = '%s/%s' % (self._local_volume_dir(volume), active_file) info = self._qemu_img_info(active_file_path) backing_file = info.backing_file if backing_file: snapshots_exist = True else: snapshots_exist = False root_file_fmt = info.file_format tmp_params = { 'prefix': '%s.temp_image.%s' % (volume['id'], image_meta['id']), 'suffix': '.img' } with image_utils.temporary_file(**tmp_params) as temp_path: if snapshots_exist or (root_file_fmt != 'raw'): # Convert due to snapshots # or volume data not being stored in raw format # (upload_volume assumes raw format input) image_utils.convert_image(active_file_path, temp_path, 'raw') upload_path = temp_path else: upload_path = active_file_path image_utils.upload_volume(context, image_service, image_meta, upload_path)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ LOG.debug( "snapshot: %(snap)s, volume: %(vol)s, ", {"snap": snapshot["id"], "vol": volume["id"], "size": volume_size} ) info_path = self._local_path_volume_info(snapshot["volume"]) snap_info = self._read_info_file(info_path) vol_path = self._local_volume_dir(snapshot["volume"]) forward_file = snap_info[snapshot["id"]] forward_path = os.path.join(vol_path, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot["volume"]["name"]) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) path_to_new_vol = self._local_path_volume(volume) LOG.debug("will copy from snapshot at %s", path_to_snap_img) if self.configuration.quobyte_qcow2_volumes: out_format = "qcow2" else: out_format = "raw" image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root) self._set_rw_permissions_for_all(path_to_new_vol)
def copy_volume_to_image(self, context, volume, image_service, image_meta): """Copy the volume to the specified image.""" # If snapshots exist, flatten to a temporary image, and upload it active_file = self.get_active_image_from_info(volume) active_file_path = "%s/%s" % (self._local_volume_dir(volume), active_file) backing_file = self._get_backing_file_for_path(active_file_path) if backing_file is not None: snapshots_exist = True else: snapshots_exist = False root_file_fmt = self._get_file_format_for_path(self._local_path_volume(volume)) temp_path = None try: if snapshots_exist or (root_file_fmt != "raw"): # Convert due to snapshots # or volume data not being stored in raw format # (upload_volume assumes raw format input) temp_path = "%s/%s.temp_image.%s" % (self._local_volume_dir(volume), volume["id"], image_meta["id"]) image_utils.convert_image(active_file_path, temp_path, "raw") upload_path = temp_path else: upload_path = active_file_path image_utils.upload_volume(context, image_service, image_meta, upload_path) finally: if temp_path is not None: self._execute("rm", "-f", temp_path)
def _direct_nfs_clone(self, volume, image_location, image_id): """Clone directly in nfs share.""" LOG.info(_("Cloning image %s directly in share"), image_id) cloned = False image_location = self._construct_image_nfs_url(image_location) share = self._is_cloneable_share(image_location) if share and self._is_share_eligible(share, volume["size"]): LOG.debug(_("Share is cloneable %s"), share) volume["provider_location"] = share (__, ___, img_file) = image_location.rpartition("/") dir_path = self._get_mount_point_for_share(share) img_path = "%s/%s" % (dir_path, img_file) img_info = image_utils.qemu_img_info(img_path) if img_info.file_format == "raw": LOG.debug(_("Image is raw %s"), image_id) self._clone_volume(img_file, volume["name"], volume_id=None, share=share) cloned = True else: LOG.info(_("Image will locally be converted to raw %s"), image_id) dst = "%s/%s" % (dir_path, volume["name"]) image_utils.convert_image(img_path, dst, "raw") data = image_utils.qemu_img_info(dst) if data.file_format != "raw": raise exception.InvalidResults(_("Converted to raw, but" " format is now %s") % data.file_format) else: cloned = True self._register_image_in_cache(volume, image_id) return cloned
def test_copy_volume_from_snapshot(self): (mox, drv) = self._mox, self._driver mox.StubOutWithMock(image_utils, 'convert_image') mox.StubOutWithMock(drv, '_read_info_file') mox.StubOutWithMock(image_utils, 'qemu_img_info') mox.StubOutWithMock(drv, '_set_rw_permissions_for_all') dest_volume = self._simple_volume( 'c1073000-0000-0000-0000-0000000c1073') src_volume = self._simple_volume() vol_dir = os.path.join(self.TEST_MNT_POINT_BASE, drv._get_hash_str(self.TEST_QUOBYTE_VOLUME)) src_vol_path = os.path.join(vol_dir, src_volume['name']) dest_vol_path = os.path.join(vol_dir, dest_volume['name']) info_path = os.path.join(vol_dir, src_volume['name']) + '.info' snapshot = {'volume_name': src_volume['name'], 'name': 'clone-snap-%s' % src_volume['id'], 'size': src_volume['size'], 'volume_size': src_volume['size'], 'volume_id': src_volume['id'], 'id': 'tmp-snap-%s' % src_volume['id'], 'volume': src_volume} snap_file = dest_volume['name'] + '.' + snapshot['id'] snap_path = os.path.join(vol_dir, snap_file) size = dest_volume['size'] drv._read_info_file(info_path).AndReturn( {'active': snap_file, snapshot['id']: snap_file} ) qemu_img_output = """image: %s file format: raw virtual size: 1.0G (1073741824 bytes) disk size: 173K backing file: %s """ % (snap_file, src_volume['name']) img_info = imageutils.QemuImgInfo(qemu_img_output) image_utils.qemu_img_info(snap_path).AndReturn(img_info) image_utils.convert_image(src_vol_path, dest_vol_path, 'raw', run_as_root=self.execute_as_root) drv._set_rw_permissions_for_all(dest_vol_path) mox.ReplayAll() drv._copy_volume_from_snapshot(snapshot, dest_volume, size) mox.VerifyAll()
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. If the quobyte_volume_from_snapshot_cache is active the result is copied into the cache and all volumes created from this snapshot id are directly copied from the cache. """ LOG.debug("snapshot: %(snap)s, volume: %(vol)s, ", {'snap': snapshot.id, 'vol': volume.id, 'size': volume_size}) info_path = self._local_path_volume_info(snapshot.volume) snap_info = self._read_info_file(info_path) vol_path = self._local_volume_dir(snapshot.volume) forward_file = snap_info[snapshot.id] forward_path = os.path.join(vol_path, forward_file) self._ensure_shares_mounted() # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot.volume.name) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) path_to_new_vol = self._local_path_volume(volume) path_to_cached_vol = self._local_volume_from_snap_cache_path(snapshot) LOG.debug("will copy from snapshot at %s", path_to_snap_img) if self.configuration.quobyte_qcow2_volumes: out_format = 'qcow2' else: out_format = 'raw' if not self.configuration.quobyte_volume_from_snapshot_cache: LOG.debug("Creating direct copy from snapshot") image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root) else: # create the volume via volume cache if not os.access(path_to_cached_vol, os.F_OK): LOG.debug("Caching volume %(volpath)s from snapshot.", {'volpath': path_to_cached_vol}) image_utils.convert_image(path_to_snap_img, path_to_cached_vol, out_format, run_as_root=self._execute_as_root) # Copy volume from cache LOG.debug("Copying volume %(volpath)s from cache", {'volpath': path_to_new_vol}) shutil.copyfile(path_to_cached_vol, path_to_new_vol) self._set_rw_permissions(path_to_new_vol) self.optimize_volume(volume)
def copy_image_to_volume(self, context, volume, image_service, image_id): with image_utils.temporary_file() as tmp: # (wenhao): we don't need to convert to raw for sheepdog. image_utils.fetch_verify_image(context, image_service, image_id, tmp) # remove the image created by import before this function. # see volume/drivers/manager.py:_create_volume self.client.delete(volume.name) # convert and store into sheepdog image_utils.convert_image(tmp, self.client.local_path(volume), "raw") self.client.resize(volume.name, volume.size)
def copy_image_to_volume(self, context, volume, image_service, image_id): with image_utils.temporary_file() as tmp: # (wenhao): we don't need to convert to raw for sheepdog. image_utils.fetch_verify_image(context, image_service, image_id, tmp) # remove the image created by import before this function. # see volume/drivers/manager.py:_create_volume self._delete(volume) # convert and store into sheepdog image_utils.convert_image(tmp, 'sheepdog:%s' % volume['name'], 'raw') self._resize(volume)
def test_convert_image(self): mox = self._mox mox.StubOutWithMock(utils, "execute") TEST_OUT_FORMAT = "vmdk" TEST_SOURCE = "img/qemu.img" TEST_DEST = "/img/vmware.vmdk" utils.execute("qemu-img", "convert", "-O", TEST_OUT_FORMAT, TEST_SOURCE, TEST_DEST, run_as_root=True) mox.ReplayAll() image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT) mox.VerifyAll()
def _clone_image(self, volume, image_location, image_id): """Attempt to create a volume by efficiently copying image to volume. If both source and target are backed by gpfs storage and the source image is in raw format move the image to create a volume using either gpfs clone operation or with a file copy. If the image format is not raw, convert it to raw at the volume path. """ # Check if GPFS is mounted self._verify_gpfs_path_state(self.configuration.gpfs_mount_point_base) cloneable_image, reason, image_path = self._is_cloneable(image_id) if not cloneable_image: LOG.debug('Image %(img)s not cloneable: %(reas)s' % {'img': image_id, 'reas': reason}) return (None, False) vol_path = self.local_path(volume) # if the image is not already a GPFS snap file make it so if not self._is_gpfs_parent_file(image_path): self._create_gpfs_snap(image_path, modebits='666') data = image_utils.qemu_img_info(image_path) # if image format is already raw either clone it or # copy it depending on config file settings if data.file_format == 'raw': if (self.configuration.gpfs_images_share_mode == 'copy_on_write'): LOG.debug('Clone image to vol %s using mmclone' % volume['id']) self._create_gpfs_copy(image_path, vol_path) elif self.configuration.gpfs_images_share_mode == 'copy': LOG.debug('Clone image to vol %s using copyfile' % volume['id']) shutil.copyfile(image_path, vol_path) self._execute('chmod', '666', vol_path, run_as_root=True) # if image is not raw convert it to raw into vol_path destination else: LOG.debug('Clone image to vol %s using qemu convert' % volume['id']) image_utils.convert_image(image_path, vol_path, 'raw') self._execute('chmod', '666', vol_path, run_as_root=True) self._resize_volume_file(volume, volume['size']) return {'provider_location': None}, True
def _direct_clone(self, volume, image_location, image_id, image_name): """Clones directly in nfs share.""" LOG.info('Checking image clone %s from glance share.', image_id) cloned = False image_location = self._get_image_nfs_url(image_location) share = self._is_cloneable_share(image_location) run_as_root = self._execute_as_root dst_share = None for dst in self._mounted_shares: if dst and self._is_share_vol_compatible(volume, dst): dst_share = dst LOG.debug('Image dst share: %s', dst) break if not dst_share: return cloned LOG.debug('Share is cloneable %s', dst_share) volume['provider_location'] = dst_share (__, ___, img_file) = image_location.rpartition('/') dir_path = self._get_mount_point_for_share(share) dst_path = self._get_mount_point_for_share(dst_share) img_path = '%s/%s' % (dir_path, img_file) img_info = image_utils.qemu_img_info(img_path, run_as_root=run_as_root) if img_info.file_format == 'raw': LOG.debug('Image is raw %s', image_id) self._clone_volume_to_volume( img_file, volume['name'], image_name, volume_id=None, share=share, dst=dst_share, image_id=image_id) cloned = True else: LOG.info('Image will locally be converted to raw %s', image_id) dst = '%s/%s' % (dst_path, volume['name']) image_utils.convert_image(img_path, dst, 'raw', run_as_root=run_as_root) data = image_utils.qemu_img_info(dst, run_as_root=run_as_root) if data.file_format != "raw": raise exception.InvalidResults( _('Converted to raw, but ' 'format is now %s') % data.file_format) else: cloned = True self._create_image_snapshot( volume['name'], volume['provider_location'], image_id, image_name) return cloned
def test_convert_image(self): mox = self._mox mox.StubOutWithMock(utils, 'execute') TEST_OUT_FORMAT = 'vmdk' TEST_SOURCE = 'img/qemu.img' TEST_DEST = '/img/vmware.vmdk' utils.execute('qemu-img', 'convert', '-O', TEST_OUT_FORMAT, TEST_SOURCE, TEST_DEST, run_as_root=True) mox.ReplayAll() image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT) mox.VerifyAll()
def copy_image_to_volume(self, context, volume, image_service, image_id): # use the image_conversion_dir as a temporary place to save the image conversion_dir = CONF.image_conversion_dir self._ensure_dir_exists(conversion_dir) with tempfile.NamedTemporaryFile(dir=conversion_dir) as tmp: # (wenhao): we don't need to convert to raw for sheepdog. image_utils.fetch_verify_image(context, image_service, image_id, tmp.name) # remove the image created by import before this function. # see volume/drivers/manager.py:_create_volume self._delete(volume) # convert and store into sheepdog image_utils.convert_image(tmp, 'sheepdog:%s' % volume['name'], 'raw') self._resize(volume)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ info_path = self._local_path_volume_info(snapshot.volume) snap_info = self._read_info_file(info_path) vol_dir = self._local_volume_dir(snapshot.volume) out_format = self.choose_volume_format(volume) volume_format = self.get_volume_format(snapshot.volume) volume_path = self.local_path(volume) if volume_format in (DISK_FORMAT_QCOW2, DISK_FORMAT_RAW): forward_file = snap_info[snapshot.id] forward_path = os.path.join(vol_dir, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot.volume.name) path_to_snap_img = os.path.join(vol_dir, img_info.backing_file) LOG.debug("_copy_volume_from_snapshot: will copy " "from snapshot at %s.", path_to_snap_img) image_utils.convert_image(path_to_snap_img, volume_path, out_format) elif volume_format == DISK_FORMAT_PLOOP: with PloopDevice(self.local_path(snapshot.volume), snapshot.id, execute=self._execute) as dev: image_utils.convert_image(dev, volume_path, out_format) else: msg = _("Unsupported volume format %s") % volume_format raise exception.InvalidVolume(msg) if out_format == DISK_FORMAT_PLOOP: img_path = os.path.join(volume_path, 'root.hds') os.rename(volume_path, volume_path + '.tmp') os.mkdir(volume_path) os.rename(volume_path + '.tmp', img_path) self._execute('ploop', 'restore-descriptor', volume_path, img_path) self._extend_volume(volume, volume_size, out_format)
def test_clone_image_resizefails(self): drv = self._driver mox = self.mox volume = {'name': 'vol', 'size': '20'} mox.StubOutWithMock(drv, '_find_image_in_cache') mox.StubOutWithMock(drv, '_is_cloneable_share') mox.StubOutWithMock(drv, '_get_mount_point_for_share') mox.StubOutWithMock(image_utils, 'qemu_img_info') mox.StubOutWithMock(drv, '_clone_volume') mox.StubOutWithMock(drv, '_discover_file_till_timeout') mox.StubOutWithMock(drv, '_set_rw_permissions_for_all') mox.StubOutWithMock(drv, '_resize_image_file') mox.StubOutWithMock(image_utils, 'convert_image') mox.StubOutWithMock(drv, '_register_image_in_cache') mox.StubOutWithMock(drv, '_is_share_vol_compatible') mox.StubOutWithMock(drv, 'local_path') mox.StubOutWithMock(os.path, 'exists') mox.StubOutWithMock(drv, '_delete_file') drv._find_image_in_cache(IgnoreArg()).AndReturn([]) drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn( '127.0.0.1:/share') drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True) drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt') image_utils.qemu_img_info('/mnt/img-id').AndReturn( self.get_img_info('notraw')) image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw') image_utils.qemu_img_info('/mnt/vol').AndReturn( self.get_img_info('raw')) drv._register_image_in_cache(IgnoreArg(), IgnoreArg()) drv.local_path(IgnoreArg()).AndReturn('/mnt/vol') drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True) drv._set_rw_permissions_for_all('/mnt/vol') drv._resize_image_file( IgnoreArg(), IgnoreArg()).AndRaise(exception.InvalidResults()) drv.local_path(IgnoreArg()).AndReturn('/mnt/vol') os.path.exists('/mnt/vol').AndReturn(True) drv._delete_file('/mnt/vol') mox.ReplayAll() vol_dict, result = drv. clone_image( volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {}) mox.VerifyAll() self.assertFalse(result) self.assertFalse(vol_dict['bootable']) self.assertIsNone(vol_dict['provider_location'])
def _gpfs_fetch_to_raw(self, context, image_service, image_id, dest, user_id=None, project_id=None): if (self.configuration.image_conversion_dir and not os.path.exists(self.configuration.image_conversion_dir)): os.makedirs(self.configuration.image_conversion_dir) tmp = "%s.part" % dest with fileutils.remove_path_on_error(tmp): self._gpfs_fetch(context, image_service, image_id, tmp, user_id, project_id) data = image_utils.qemu_img_info(tmp) fmt = data.file_format backing_file = data.backing_file if backing_file is not None: msg = (_("fmt = %(fmt)s backed by: %(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file}) LOG.error(msg) raise exception.ImageUnacceptable( image_id=image_id, reason=msg) if fmt is None: msg = _("'qemu-img info' parsing failed.") LOG.error(msg) raise exception.ImageUnacceptable( reason=msg, image_id=image_id) elif fmt == 'raw': # already in raw format - just rename to dest self._execute('mv', tmp, dest, run_as_root=True) else: # conversion to raw format required LOG.debug("%s was %s, converting to raw" % (image_id, fmt)) image_utils.convert_image(tmp, dest, 'raw') os.unlink(tmp) data = image_utils.qemu_img_info(dest) if data.file_format != "raw": msg = (_("Expected image to be in raw format, but is %s") % data.file_format) LOG.error(msg) raise exception.ImageUnacceptable( image_id=image_id, reason=msg) return {'size': math.ceil(data.virtual_size / 1024.0 ** 3)}
def _do_extend_volume(self, volume_path, size_gb, volume_name): info = self._qemu_img_info(volume_path, volume_name) fmt = info.file_format # Note(lpetrut): as for version 2.0, qemu-img cannot resize # vhd/x images. For the moment, we'll just use an intermediary # conversion in order to be able to do the resize. if fmt in (self._DISK_FORMAT_VHDX, self._DISK_FORMAT_VHD_LEGACY): temp_image = volume_path + ".tmp" image_utils.convert_image(volume_path, temp_image, self._DISK_FORMAT_RAW) image_utils.resize_image(temp_image, size_gb) image_utils.convert_image(temp_image, volume_path, fmt) self._delete(temp_image) else: image_utils.resize_image(volume_path, size_gb) if not self._is_file_size_equal(volume_path, size_gb): raise exception.ExtendVolumeError(reason="Resizing image file failed.")
def copy_image_to_volume(self, context, volume, image_service, image_id): with image_utils.temporary_file() as tmp: # (wenhao): we don't need to convert to raw for sheepdog. image_utils.fetch_verify_image(context, image_service, image_id, tmp) # remove the image created by import before this function. # see volume/drivers/manager.py:_create_volume self.client.delete(volume.name) # convert and store into sheepdog image_utils.convert_image( tmp, 'sheepdog:%(addr)s:%(port)d:%(name)s' % { 'addr': CONF.sheepdog_store_address, 'port': CONF.sheepdog_store_port, 'name': volume['name']}, 'raw') self.client.resize(volume.name, volume.size)
def _clone_image(self, volume, image_location, image_id): """Attempt to create a volume by efficiently copying image to volume. If both source and target are backed by gpfs storage and the source image is in raw format move the image to create a volume using either gpfs clone operation or with a file copy. If the image format is not raw, convert it to raw at the volume path. """ # Check if GPFS is mounted self._verify_gpfs_path_state(self.configuration.gpfs_mount_point_base) cloneable_image, reason, image_path = self._is_cloneable(image_id) if not cloneable_image: LOG.debug("Image %(img)s not cloneable: %(reas)s." % {"img": image_id, "reas": reason}) return (None, False) vol_path = self.local_path(volume) data = image_utils.qemu_img_info(image_path) # if image format is already raw either clone it or # copy it depending on config file settings if data.file_format == "raw": if self.configuration.gpfs_images_share_mode == "copy_on_write": LOG.debug("Clone image to vol %s using mmclone." % volume["id"]) # if the image is not already a GPFS snap file make it so if not self._is_gpfs_parent_file(image_path): self._create_gpfs_snap(image_path) self._create_gpfs_copy(image_path, vol_path) elif self.configuration.gpfs_images_share_mode == "copy": LOG.debug("Clone image to vol %s using copyfile." % volume["id"]) shutil.copyfile(image_path, vol_path) # if image is not raw convert it to raw into vol_path destination else: LOG.debug("Clone image to vol %s using qemu convert." % volume["id"]) image_utils.convert_image(image_path, vol_path, "raw") self._set_rw_permission(vol_path) self._resize_volume_file(volume, volume["size"]) return {"provider_location": None}, True
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ info_path = self._local_path_volume_info(snapshot.volume) snap_info = self._read_info_file(info_path) vol_dir = self._local_volume_dir(snapshot.volume) out_format = self.choose_volume_format(volume) qemu_out_format = image_utils.fixup_disk_format(out_format) volume_format = self.get_volume_format(snapshot.volume) volume_path = self.local_path(volume) if volume_format in (DISK_FORMAT_QCOW2, DISK_FORMAT_RAW): forward_file = snap_info[snapshot.id] forward_path = os.path.join(vol_dir, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot.volume.name) path_to_snap_img = os.path.join(vol_dir, img_info.backing_file) LOG.debug( "_copy_volume_from_snapshot: will copy " "from snapshot at %s.", path_to_snap_img) image_utils.convert_image(path_to_snap_img, volume_path, qemu_out_format) elif volume_format == DISK_FORMAT_PLOOP: with PloopDevice(self.local_path(snapshot.volume), snapshot.id, execute=self._execute) as dev: base_file = os.path.join(volume_path, 'root.hds') image_utils.convert_image(dev, base_file, qemu_out_format) else: msg = _("Unsupported volume format %s") % volume_format raise exception.InvalidVolume(msg) self._extend_volume(volume, volume_size, out_format) # Query qemu-img info to cache the output img_info = self._qemu_img_info(volume_path, volume.name)
def _direct_nfs_clone(self, volume, image_location, image_id): """Clone directly in nfs share.""" LOG.info(_LI('Checking image clone %s from glance share.'), image_id) cloned = False image_locations = self._construct_image_nfs_url(image_location) run_as_root = self._execute_as_root for loc in image_locations: share = self._is_cloneable_share(loc) if share and self._is_share_vol_compatible(volume, share): LOG.debug('Share is cloneable %s', share) volume['provider_location'] = share (__, ___, img_file) = loc.rpartition('/') dir_path = self._get_mount_point_for_share(share) img_path = '%s/%s' % (dir_path, img_file) img_info = image_utils.qemu_img_info(img_path, run_as_root=run_as_root) if img_info.file_format == 'raw': LOG.debug('Image is raw %s', image_id) self._clone_backing_file_for_volume(img_file, volume['name'], volume_id=None, share=share) cloned = True break else: LOG.info(_LI('Image will locally be converted to raw %s'), image_id) dst = '%s/%s' % (dir_path, volume['name']) image_utils.convert_image(img_path, dst, 'raw', run_as_root=run_as_root) data = image_utils.qemu_img_info(dst, run_as_root=run_as_root) if data.file_format != "raw": raise exception.InvalidResults( _("Converted to raw, but" " format is now %s") % data.file_format) else: cloned = True self._register_image_in_cache(volume, image_id) break return cloned
def _direct_nfs_clone(self, volume, image_location, image_id): """Clone directly in nfs share.""" LOG.info(_LI('Checking image clone %s from glance share.'), image_id) cloned = False image_locations = self._construct_image_nfs_url(image_location) run_as_root = self._execute_as_root for loc in image_locations: share = self._is_cloneable_share(loc) if share and self._is_share_vol_compatible(volume, share): LOG.debug('Share is cloneable %s', share) volume['provider_location'] = share (__, ___, img_file) = loc.rpartition('/') dir_path = self._get_mount_point_for_share(share) img_path = '%s/%s' % (dir_path, img_file) img_info = image_utils.qemu_img_info(img_path, run_as_root=run_as_root) if img_info.file_format == 'raw': LOG.debug('Image is raw %s', image_id) self._clone_backing_file_for_volume( img_file, volume['name'], volume_id=None, share=share) cloned = True break else: LOG.info( _LI('Image will locally be converted to raw %s'), image_id) dst = '%s/%s' % (dir_path, volume['name']) image_utils.convert_image(img_path, dst, 'raw', run_as_root=run_as_root) data = image_utils.qemu_img_info(dst, run_as_root=run_as_root) if data.file_format != "raw": raise exception.InvalidResults( _("Converted to raw, but" " format is now %s") % data.file_format) else: cloned = True self._register_image_in_cache( volume, image_id) break return cloned
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ info_path = self._local_path_volume_info(snapshot['volume']) # For BC compat' with version < 2 of this driver try: snap_info = self._read_info_file(info_path) except IOError as exc: if exc.errno != errno.ENOENT: raise else: path_to_snap_img = self.local_path(snapshot) else: vol_path = self._local_volume_dir(snapshot['volume']) forward_file = snap_info[snapshot['id']] forward_path = os.path.join(vol_path, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot['volume']['name']) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) LOG.debug("will copy from snapshot at %s", path_to_snap_img) path_to_new_vol = self.local_path(volume) out_format = 'raw' image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root) self._set_rw_permissions_for_all(path_to_new_vol) image_utils.resize_image(path_to_new_vol, volume_size)
def _do_extend_volume(self, volume_path, size_gb, volume_name): info = self._qemu_img_info(volume_path, volume_name) fmt = info.file_format # Note(lpetrut): as for version 2.0, qemu-img cannot resize # vhd/x images. For the moment, we'll just use an intermediary # conversion in order to be able to do the resize. if fmt in (self._DISK_FORMAT_VHDX, self._DISK_FORMAT_VHD_LEGACY): temp_image = volume_path + '.tmp' image_utils.convert_image(volume_path, temp_image, self._DISK_FORMAT_RAW) image_utils.resize_image(temp_image, size_gb) image_utils.convert_image(temp_image, volume_path, fmt) self._delete(temp_image) else: image_utils.resize_image(volume_path, size_gb) if not self._is_file_size_equal(volume_path, size_gb): raise exception.ExtendVolumeError( reason='Resizing image file failed.')
def copy_volume_to_image(self, context, volume, image_service, image_meta): """Copy the volume to the specified image.""" # If snapshots exist, flatten to a temporary image, and upload it active_file = self.get_active_image_from_info(volume) active_file_path = '%s/%s' % (self._local_volume_dir(volume), active_file) info = self._qemu_img_info(active_file_path, volume['name']) backing_file = info.backing_file if backing_file: snapshots_exist = True else: snapshots_exist = False root_file_fmt = info.file_format temp_path = None try: if snapshots_exist or (root_file_fmt != 'raw'): # Convert due to snapshots # or volume data not being stored in raw format # (upload_volume assumes raw format input) temp_path = '%s/%s.temp_image.%s' % ( self._local_volume_dir(volume), volume['id'], image_meta['id']) image_utils.convert_image(active_file_path, temp_path, 'raw') upload_path = temp_path else: upload_path = active_file_path image_utils.upload_volume(context, image_service, image_meta, upload_path) finally: if temp_path is not None: self._execute('rm', '-f', temp_path)
def copy_volume_to_image(self, context, volume, image_service, image_meta): """Copy the volume to the specified image.""" # If snapshots exist, flatten to a temporary image, and upload it active_file = self.get_active_image_from_info(volume) active_file_path = '%s/%s' % (self._local_volume_dir(volume), active_file) info = self._qemu_img_info(active_file_path) backing_file = info.backing_file if backing_file: snapshots_exist = True else: snapshots_exist = False root_file_fmt = info.file_format temp_path = None try: if snapshots_exist or (root_file_fmt != 'raw'): # Convert due to snapshots # or volume data not being stored in raw format # (upload_volume assumes raw format input) temp_path = '%s/%s.temp_image.%s' % ( self._local_volume_dir(volume), volume['id'], image_meta['id']) image_utils.convert_image(active_file_path, temp_path, 'raw') upload_path = temp_path else: upload_path = active_file_path image_utils.upload_volume(context, image_service, image_meta, upload_path) finally: if temp_path is not None: self._execute('rm', '-f', temp_path)
def test_clone_image_file_not_discovered(self): drv = self._driver mox = self.mox volume = {'name': 'vol', 'size': '20'} mox.StubOutWithMock(drv, '_find_image_in_cache') mox.StubOutWithMock(drv, '_is_cloneable_share') mox.StubOutWithMock(drv, '_get_mount_point_for_share') mox.StubOutWithMock(image_utils, 'qemu_img_info') mox.StubOutWithMock(drv, '_clone_volume') mox.StubOutWithMock(drv, '_discover_file_till_timeout') mox.StubOutWithMock(image_utils, 'convert_image') mox.StubOutWithMock(drv, '_register_image_in_cache') mox.StubOutWithMock(drv, '_is_share_vol_compatible') mox.StubOutWithMock(drv, 'local_path') mox.StubOutWithMock(os.path, 'exists') mox.StubOutWithMock(drv, '_delete_file') drv._find_image_in_cache(IgnoreArg()).AndReturn([]) drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn( '127.0.0.1:/share') drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True) drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt') image_utils.qemu_img_info('/mnt/img-id').AndReturn( self.get_img_info('notraw')) image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw') image_utils.qemu_img_info('/mnt/vol').AndReturn( self.get_img_info('raw')) drv._register_image_in_cache(IgnoreArg(), IgnoreArg()) drv.local_path(IgnoreArg()).AndReturn('/mnt/vol') drv._discover_file_till_timeout(IgnoreArg()).AndReturn(False) drv.local_path(IgnoreArg()).AndReturn('/mnt/vol') os.path.exists('/mnt/vol').AndReturn(True) drv._delete_file('/mnt/vol') mox.ReplayAll() vol_dict, result = drv.clone_image( volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {}) mox.VerifyAll() self.assertFalse(result) self.assertFalse(vol_dict['bootable']) self.assertIsNone(vol_dict['provider_location'])
def test_clone_image_resizefails(self): drv = self._driver mox = self.mox volume = {"name": "vol", "size": "20"} mox.StubOutWithMock(drv, "_find_image_in_cache") mox.StubOutWithMock(drv, "_is_cloneable_share") mox.StubOutWithMock(drv, "_get_mount_point_for_share") mox.StubOutWithMock(image_utils, "qemu_img_info") mox.StubOutWithMock(drv, "_clone_volume") mox.StubOutWithMock(drv, "_discover_file_till_timeout") mox.StubOutWithMock(drv, "_set_rw_permissions_for_all") mox.StubOutWithMock(drv, "_resize_image_file") mox.StubOutWithMock(image_utils, "convert_image") mox.StubOutWithMock(drv, "_register_image_in_cache") mox.StubOutWithMock(drv, "_is_share_vol_compatible") mox.StubOutWithMock(drv, "local_path") mox.StubOutWithMock(os.path, "exists") mox.StubOutWithMock(drv, "_delete_file") drv._find_image_in_cache(IgnoreArg()).AndReturn([]) drv._is_cloneable_share("nfs://127.0.0.1/share/img-id").AndReturn("127.0.0.1:/share") drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True) drv._get_mount_point_for_share("127.0.0.1:/share").AndReturn("/mnt") image_utils.qemu_img_info("/mnt/img-id").AndReturn(self.get_img_info("notraw")) image_utils.convert_image(IgnoreArg(), IgnoreArg(), "raw") image_utils.qemu_img_info("/mnt/vol").AndReturn(self.get_img_info("raw")) drv._register_image_in_cache(IgnoreArg(), IgnoreArg()) drv.local_path(IgnoreArg()).AndReturn("/mnt/vol") drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True) drv._set_rw_permissions_for_all("/mnt/vol") drv._resize_image_file(IgnoreArg(), IgnoreArg()).AndRaise(exception.InvalidResults()) drv.local_path(IgnoreArg()).AndReturn("/mnt/vol") os.path.exists("/mnt/vol").AndReturn(True) drv._delete_file("/mnt/vol") mox.ReplayAll() vol_dict, result = drv.clone_image(volume, ("nfs://127.0.0.1/share/img-id", None), "image_id") mox.VerifyAll() self.assertFalse(result) self.assertFalse(vol_dict["bootable"]) self.assertIsNone(vol_dict["provider_location"])
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ LOG.debug("snapshot: %(snap)s, volume: %(vol)s, " "volume_size: %(size)s" % { 'snap': snapshot['id'], 'vol': volume['id'], 'size': volume_size }) info_path = self._local_path_volume_info(snapshot['volume']) snap_info = self._read_info_file(info_path) vol_path = self._local_volume_dir(snapshot['volume']) forward_file = snap_info[snapshot['id']] forward_path = os.path.join(vol_path, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot['volume']['name']) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) path_to_new_vol = self._local_path_volume(volume) LOG.debug("will copy from snapshot at %s" % path_to_snap_img) if self.configuration.quobyte_qcow2_volumes: out_format = 'qcow2' else: out_format = 'raw' image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root) self._set_rw_permissions_for_all(path_to_new_vol)
def test_convert_image(self, mock_stat): mox = self._mox mox.StubOutWithMock(utils, "execute") mox.StubOutWithMock(utils, "is_blk_device") TEST_OUT_FORMAT = "vmdk" TEST_SOURCE = "img/qemu.img" TEST_DEST = "/img/vmware.vmdk" utils.is_blk_device(TEST_DEST).AndReturn(True) utils.execute("dd", "count=0", "if=img/qemu.img", "of=/img/vmware.vmdk", "oflag=direct", run_as_root=True) utils.execute( "qemu-img", "convert", "-t", "none", "-O", TEST_OUT_FORMAT, TEST_SOURCE, TEST_DEST, run_as_root=True ) mox.ReplayAll() image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT, run_as_root=True) mox.VerifyAll()
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ LOG.debug("snapshot: %(snap)s, volume: %(vol)s, " "volume_size: %(size)s" % {'snap': snapshot['id'], 'vol': volume['id'], 'size': volume_size}) info_path = self._local_path_volume_info(snapshot['volume']) snap_info = self._read_info_file(info_path) vol_path = self._local_volume_dir(snapshot['volume']) forward_file = snap_info[snapshot['id']] forward_path = os.path.join(vol_path, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot['volume']['name']) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) path_to_new_vol = self._local_path_volume(volume) LOG.debug("will copy from snapshot at %s" % path_to_snap_img) if self.configuration.glusterfs_qcow2_volumes: out_format = 'qcow2' else: out_format = 'raw' image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format) self._set_rw_permissions_for_all(path_to_new_vol)
def copy_image_to_volume(self, context, volume, image_service, image_id): """Fetch the image from image_service and write it to the volume.""" img_cache_volume = {'name': 'img-%s' % image_id} img_cache_snapshot = {'volume_name': img_cache_volume['name'], 'name': 'snap'} if self._list_snapshots(volume): # If the volume has snapshots, there is no efficient way to replace # the contents. Do things the inefficient way. image_utils.fetch_to_raw(context, image_service, image_id, self.local_path(volume), self.configuration.volume_dd_blocksize, size=volume['size']) else: if not self.snapshot_exists(img_cache_snapshot): with image_utils.temporary_file() as tmp: image_utils.fetch_verify_image(context, image_service, image_id, tmp) qemu_info = image_utils.qemu_img_info(tmp) virtual_size_gb = math.ceil( qemu_info.virtual_size / (1024.0 ** 3)) img_cache_volume['size'] = virtual_size_gb self.create_volume(img_cache_volume) image_utils.convert_image( tmp, self.local_path(img_cache_volume), 'raw', None, sparse=64 * 1024) self.create_snapshot(img_cache_snapshot) self.delete_volume(volume) self.create_volume_from_snapshot(volume, img_cache_snapshot) self.extend_volume(volume, volume['size'])
def test_convert_image(self, mock_stat): mox = self._mox mox.StubOutWithMock(utils, 'execute') mox.StubOutWithMock(utils, 'is_blk_device') TEST_OUT_FORMAT = 'vmdk' TEST_SOURCE = 'img/qemu.img' TEST_DEST = '/img/vmware.vmdk' utils.is_blk_device(TEST_DEST).AndReturn(True) utils.execute('dd', 'count=0', 'if=img/qemu.img', 'of=/img/vmware.vmdk', 'oflag=direct', run_as_root=True) utils.execute( 'qemu-img', 'convert', '-t', 'none', '-O', TEST_OUT_FORMAT, TEST_SOURCE, TEST_DEST, run_as_root=True) mox.ReplayAll() image_utils.convert_image(TEST_SOURCE, TEST_DEST, TEST_OUT_FORMAT) mox.VerifyAll()
def test_clone_image_cloneableshare_notraw(self): drv = self._driver mox = self.mox volume = {'name': 'vol', 'size': '20'} mox.StubOutWithMock(drv, '_find_image_in_cache') mox.StubOutWithMock(drv, '_is_cloneable_share') mox.StubOutWithMock(drv, '_get_mount_point_for_share') mox.StubOutWithMock(image_utils, 'qemu_img_info') mox.StubOutWithMock(drv, '_clone_volume') mox.StubOutWithMock(drv, '_discover_file_till_timeout') mox.StubOutWithMock(drv, '_set_rw_permissions_for_all') mox.StubOutWithMock(drv, '_resize_image_file') mox.StubOutWithMock(image_utils, 'convert_image') mox.StubOutWithMock(drv, '_register_image_in_cache') mox.StubOutWithMock(drv, '_is_share_vol_compatible') drv._find_image_in_cache(IgnoreArg()).AndReturn([]) drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn( '127.0.0.1:/share') drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True) drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt') image_utils.qemu_img_info('/mnt/img-id').AndReturn( self.get_img_info('notraw')) image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw') image_utils.qemu_img_info('/mnt/vol').AndReturn( self.get_img_info('raw')) drv._register_image_in_cache(IgnoreArg(), IgnoreArg()) drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt') drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True) drv._set_rw_permissions_for_all('/mnt/vol') drv._resize_image_file({'name': 'vol'}, IgnoreArg()) mox.ReplayAll() drv.clone_image(volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {}) mox.VerifyAll()
def _copy_volume_to_image(self, context, volume, image_service, image_meta): """Copy the volume to the specified image.""" # If snapshots exist, flatten to a temporary image, and upload it active_file = self.get_active_image_from_info(volume) active_file_path = os.path.join(self._local_volume_dir(volume), active_file) info = self._qemu_img_info(active_file_path, volume["name"]) backing_file = info.backing_file root_file_fmt = info.file_format tmp_params = {"prefix": "%s.temp_image.%s" % (volume["id"], image_meta["id"]), "suffix": ".img"} with image_utils.temporary_file(**tmp_params) as temp_path: if backing_file or (root_file_fmt != "raw"): # Convert due to snapshots # or volume data not being stored in raw format # (upload_volume assumes raw format input) image_utils.convert_image(active_file_path, temp_path, "raw") upload_path = temp_path else: upload_path = active_file_path image_utils.upload_volume(context, image_service, image_meta, upload_path)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ LOG.debug( "Snapshot: %(snap)s, volume: %(vol)s, " "volume_size: %(size)s", { 'snap': snapshot['id'], 'vol': volume['id'], 'size': volume_size }) info_path = self._local_path_volume_info(snapshot['volume']) snap_info = self._read_info_file(info_path) vol_dir = self._local_volume_dir(snapshot['volume']) out_format = self.get_volume_format(volume, qemu_format=True) forward_file = snap_info[snapshot['id']] forward_path = os.path.join(vol_dir, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot['volume']['name']) path_to_snap_img = os.path.join(vol_dir, img_info.backing_file) LOG.debug("Will copy from snapshot at %s", path_to_snap_img) image_utils.convert_image(path_to_snap_img, self.local_path(volume), out_format) self._extend_volume(volume, volume_size) self._set_rw_permissions_for_all(self.local_path(volume))
def test_clone_image_cloneableshare_notraw(self): drv = self._driver mox = self.mox volume = {'name': 'vol', 'size': '20'} mox.StubOutWithMock(drv, '_find_image_in_cache') mox.StubOutWithMock(drv, '_is_cloneable_share') mox.StubOutWithMock(drv, '_get_mount_point_for_share') mox.StubOutWithMock(image_utils, 'qemu_img_info') mox.StubOutWithMock(drv, '_clone_volume') mox.StubOutWithMock(drv, '_discover_file_till_timeout') mox.StubOutWithMock(drv, '_set_rw_permissions_for_all') mox.StubOutWithMock(drv, '_resize_image_file') mox.StubOutWithMock(image_utils, 'convert_image') mox.StubOutWithMock(drv, '_register_image_in_cache') mox.StubOutWithMock(drv, '_is_share_vol_compatible') drv._find_image_in_cache(IgnoreArg()).AndReturn([]) drv._is_cloneable_share('nfs://127.0.0.1/share/img-id').AndReturn( '127.0.0.1:/share') drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(True) drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt') image_utils.qemu_img_info('/mnt/img-id').AndReturn( self.get_img_info('notraw')) image_utils.convert_image(IgnoreArg(), IgnoreArg(), 'raw') image_utils.qemu_img_info('/mnt/vol').AndReturn( self.get_img_info('raw')) drv._register_image_in_cache(IgnoreArg(), IgnoreArg()) drv._get_mount_point_for_share('127.0.0.1:/share').AndReturn('/mnt') drv._discover_file_till_timeout(IgnoreArg()).AndReturn(True) drv._set_rw_permissions_for_all('/mnt/vol') drv._resize_image_file({'name': 'vol'}, IgnoreArg()) mox.ReplayAll() drv. clone_image( volume, ('nfs://127.0.0.1/share/img-id', None), 'image_id', {}) mox.VerifyAll()
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size, src_encryption_key_id=None, new_encryption_key_id=None): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. """ LOG.debug("Copying snapshot: %(snap)s -> volume: %(vol)s, " "volume_size: %(size)s GB", {'snap': snapshot.id, 'vol': volume.id, 'size': volume_size}) info_path = self._local_path_volume_info(snapshot.volume) snap_info = self._read_info_file(info_path) vol_path = self._local_volume_dir(snapshot.volume) forward_file = snap_info[snapshot.id] forward_path = os.path.join(vol_path, forward_file) # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot.volume.name) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) path_to_new_vol = self._local_path_volume(volume) LOG.debug("will copy from snapshot at %s", path_to_snap_img) if self.configuration.nfs_qcow2_volumes: out_format = 'qcow2' else: out_format = 'raw' if new_encryption_key_id is not None: if src_encryption_key_id is None: message = _("Can't create an encrypted volume %(format)s " "from an unencrypted source." ) % {'format': out_format} LOG.error(message) # TODO(enriquetaso): handle unencrypted snap->encrypted vol raise exception.NfsException(message) keymgr = key_manager.API(CONF) new_key = keymgr.get(volume.obj_context, new_encryption_key_id) new_passphrase = \ binascii.hexlify(new_key.get_encoded()).decode('utf-8') # volume.obj_context is the owner of this request src_key = keymgr.get(volume.obj_context, src_encryption_key_id) src_passphrase = \ binascii.hexlify(src_key.get_encoded()).decode('utf-8') tmp_dir = volume_utils.image_conversion_dir() with tempfile.NamedTemporaryFile(prefix='luks_', dir=tmp_dir) as src_pass_file: with open(src_pass_file.name, 'w') as f: f.write(src_passphrase) with tempfile.NamedTemporaryFile(prefix='luks_', dir=tmp_dir) as new_pass_file: with open(new_pass_file.name, 'w') as f: f.write(new_passphrase) image_utils.convert_image( path_to_snap_img, path_to_new_vol, 'luks', passphrase_file=new_pass_file.name, src_passphrase_file=src_pass_file.name, run_as_root=self._execute_as_root) else: image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root) self._set_rw_permissions_for_all(path_to_new_vol)
def _cache_vol_2_2(self, context, vol, image_meta, image_service): image_id = image_meta['id'] # Pull down image and determine if valid with image_utils.TemporaryImages.fetch(image_service, context, image_id) as tmp_image: data = image_utils.qemu_img_info(tmp_image) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % { 'fmt': fmt, 'backing_file': backing_file, }) vsize = int(math.ceil(float(data.virtual_size) / units.Gi)) vol['size'] = vsize vtype = vol['volume_type_id'] LOG.info( "Creating cached image with volume type: %(vtype)s and " "size %(size)s", { 'vtype': vtype, 'size': vsize }) self._create_volume_2_2(vol) with self._connect_vol(context, vol) as device: LOG.debug("Moving image %s to volume %s", image_meta['id'], datc.get_name(vol)) image_utils.convert_image(tmp_image, device, 'raw', run_as_root=True) LOG.debug("Finished moving image %s to volume %s", image_meta['id'], datc.get_name(vol)) data = image_utils.qemu_img_info(device, run_as_root=True) if data.file_format != 'raw': raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(vol_format)s, but format is " "now %(file_format)s") % { 'vol_format': 'raw', 'file_format': data.file_format }) # TODO(_alastor_): Remove this snapshot creation when we fix # "created_at" attribute in the frontend # We don't actually care about the snapshot uuid, we just want # a single snapshot snapshot = { 'id': str(uuid.uuid4()), 'volume_id': vol['id'], 'project_id': vol['project_id'] } self._create_snapshot_2_2(snapshot) metadata = {'type': 'cached_image'} tenant = self.get_tenant(vol['project_id']) ai = self.cvol_to_ai(vol, tenant=tenant) ai.metadata.set(tenant=tenant, **metadata) # Cloning offline AI is ~4 seconds faster than cloning online AI self._detach_volume_2_2(None, vol)
def _copy_from_img_service(self, context, volume, image_service, image_id): """Copies from the image service using copy offload.""" LOG.debug("Trying copy from image service using copy offload.") image_loc = image_service.get_location(context, image_id) locations = self._construct_image_nfs_url(image_loc) src_ip = None selected_loc = None cloned = False # this will match the first location that has a valid IP on cluster for location in locations: conn, dr = self._check_get_nfs_path_segs(location) if conn: try: src_ip = self._get_ip_verify_on_cluster(conn.split(':')[0]) selected_loc = location break except exception.NotFound: pass if src_ip is None: raise exception.NotFound(_("Source host details not found.")) (__, ___, img_file) = selected_loc.rpartition('/') src_path = os.path.join(dr, img_file) dst_ip, vol_path = self._get_destination_ip_and_path(volume) share_path = vol_path.rsplit("/", 1)[0] dst_share = dst_ip + ':' + share_path # tmp file is required to deal with img formats tmp_img_file = six.text_type(uuid.uuid4()) col_path = self.configuration.netapp_copyoffload_tool_path img_info = image_service.show(context, image_id) self._check_share_can_hold_size(dst_share, img_info['size']) run_as_root = self._execute_as_root dst_dir = self._get_mount_point_for_share(dst_share) dst_img_local = os.path.join(dst_dir, tmp_img_file) try: dst_img_serv_path = os.path.join(share_path, tmp_img_file) # Always run copy offload as regular user, it's sufficient # and rootwrap doesn't allow copy offload to run as root # anyways. self._execute(col_path, src_ip, dst_ip, src_path, dst_img_serv_path, run_as_root=False, check_exit_code=0) self._discover_file_till_timeout(dst_img_local, timeout=120) LOG.debug('Copied image %(img)s to tmp file %(tmp)s.', { 'img': image_id, 'tmp': tmp_img_file }) dst_img_cache_local = os.path.join(dst_dir, 'img-cache-%s' % image_id) if img_info['disk_format'] == 'raw': LOG.debug('Image is raw %s.', image_id) self._clone_file_dst_exists(dst_share, tmp_img_file, volume['name'], dest_exists=True) self._move_nfs_file(dst_img_local, dst_img_cache_local) LOG.debug('Copied raw image %(img)s to volume %(vol)s.', { 'img': image_id, 'vol': volume['id'] }) else: LOG.debug('Image will be converted to raw %s.', image_id) img_conv = six.text_type(uuid.uuid4()) dst_img_conv_local = os.path.join(dst_dir, img_conv) # Checking against image size which is approximate check self._check_share_can_hold_size(dst_share, img_info['size']) try: image_utils.convert_image(dst_img_local, dst_img_conv_local, 'raw', run_as_root=run_as_root) data = image_utils.qemu_img_info(dst_img_conv_local, run_as_root=run_as_root) if data.file_format != "raw": raise exception.InvalidResults( _("Converted to raw, but format is now %s.") % data.file_format) else: self._clone_file_dst_exists(dst_share, img_conv, volume['name'], dest_exists=True) self._move_nfs_file(dst_img_conv_local, dst_img_cache_local) LOG.debug( 'Copied locally converted raw image' ' %(img)s to volume %(vol)s.', { 'img': image_id, 'vol': volume['id'] }) finally: if os.path.exists(dst_img_conv_local): self._delete_file_at_path(dst_img_conv_local) cloned = True finally: if os.path.exists(dst_img_local): self._delete_file_at_path(dst_img_local) return cloned
def _copy_from_img_service(self, context, volume, image_service, image_id): """Copies from the image service using copy offload.""" LOG.debug("Trying copy from image service using copy offload.") image_loc = image_service.get_location(context, image_id) image_loc = self._construct_image_nfs_url(image_loc) conn, dr = self._check_get_nfs_path_segs(image_loc) if conn: src_ip = self._get_ip_verify_on_cluster(conn.split(':')[0]) else: raise exception.NotFound(_("Source host details not found.")) (__, ___, img_file) = image_loc.rpartition('/') src_path = os.path.join(dr, img_file) dst_ip = self._get_ip_verify_on_cluster(self._get_host_ip( volume['id'])) # tmp file is required to deal with img formats tmp_img_file = six.text_type(uuid.uuid4()) col_path = self.configuration.netapp_copyoffload_tool_path img_info = image_service.show(context, image_id) dst_share = self._get_provider_location(volume['id']) self._check_share_can_hold_size(dst_share, img_info['size']) run_as_root = self._execute_as_root dst_dir = self._get_mount_point_for_share(dst_share) dst_img_local = os.path.join(dst_dir, tmp_img_file) try: # If src and dst share not equal if (('%s:%s' % (src_ip, dr)) != ('%s:%s' % (dst_ip, self._get_export_path(volume['id'])))): dst_img_serv_path = os.path.join( self._get_export_path(volume['id']), tmp_img_file) self._execute(col_path, src_ip, dst_ip, src_path, dst_img_serv_path, run_as_root=run_as_root, check_exit_code=0) else: self._clone_file_dst_exists(dst_share, img_file, tmp_img_file) self._discover_file_till_timeout(dst_img_local, timeout=120) LOG.debug('Copied image %(img)s to tmp file %(tmp)s.', {'img': image_id, 'tmp': tmp_img_file}) dst_img_cache_local = os.path.join(dst_dir, 'img-cache-%s' % image_id) if img_info['disk_format'] == 'raw': LOG.debug('Image is raw %s.', image_id) self._clone_file_dst_exists(dst_share, tmp_img_file, volume['name'], dest_exists=True) self._move_nfs_file(dst_img_local, dst_img_cache_local) LOG.debug('Copied raw image %(img)s to volume %(vol)s.', {'img': image_id, 'vol': volume['id']}) else: LOG.debug('Image will be converted to raw %s.', image_id) img_conv = six.text_type(uuid.uuid4()) dst_img_conv_local = os.path.join(dst_dir, img_conv) # Checking against image size which is approximate check self._check_share_can_hold_size(dst_share, img_info['size']) try: image_utils.convert_image(dst_img_local, dst_img_conv_local, 'raw', run_as_root=run_as_root) data = image_utils.qemu_img_info(dst_img_conv_local, run_as_root=run_as_root) if data.file_format != "raw": raise exception.InvalidResults( _("Converted to raw, but format is now %s.") % data.file_format) else: self._clone_file_dst_exists(dst_share, img_conv, volume['name'], dest_exists=True) self._move_nfs_file(dst_img_conv_local, dst_img_cache_local) LOG.debug('Copied locally converted raw image' ' %(img)s to volume %(vol)s.', {'img': image_id, 'vol': volume['id']}) finally: if os.path.exists(dst_img_conv_local): self._delete_file_at_path(dst_img_conv_local) self._post_clone_image(volume) finally: if os.path.exists(dst_img_local): self._delete_file_at_path(dst_img_local)
def _copy_volume_from_snapshot(self, snapshot, volume, volume_size): """Copy data from snapshot to destination volume. This is done with a qemu-img convert to raw/qcow2 from the snapshot qcow2. If the quobyte_volume_from_snapshot_cache is active the result is written into the cache and all volumes created from this snapshot id are created directly from the cache. """ LOG.debug("snapshot: %(snap)s, volume: %(vol)s, ", { 'snap': snapshot.id, 'vol': volume.id, 'size': volume_size }) info_path = self._local_path_volume_info(snapshot.volume) snap_info = self._read_info_file(info_path) vol_path = self._local_volume_dir(snapshot.volume) forward_file = snap_info[snapshot.id] forward_path = os.path.join(vol_path, forward_file) self._ensure_shares_mounted() # Find the file which backs this file, which represents the point # when this snapshot was created. img_info = self._qemu_img_info(forward_path, snapshot.volume.name) path_to_snap_img = os.path.join(vol_path, img_info.backing_file) path_to_new_vol = self._local_path_volume(volume) path_to_cached_vol = self._local_volume_from_snap_cache_path(snapshot) LOG.debug("will copy from snapshot at %s", path_to_snap_img) if self.configuration.quobyte_qcow2_volumes: out_format = 'qcow2' else: out_format = 'raw' if not self.configuration.quobyte_volume_from_snapshot_cache: LOG.debug("Creating direct copy from snapshot") image_utils.convert_image(path_to_snap_img, path_to_new_vol, out_format, run_as_root=self._execute_as_root) else: # create the volume via volume cache if not os.access(path_to_cached_vol, os.F_OK): LOG.debug("Caching volume %(volpath)s from snapshot.", {'volpath': path_to_cached_vol}) image_utils.convert_image(path_to_snap_img, path_to_cached_vol, out_format, run_as_root=self._execute_as_root) if self.configuration.quobyte_overlay_volumes: # NOTE(kaisers): Create a parent symlink to track the # existence of the parent os.symlink(path_to_snap_img, path_to_cached_vol + '.parent-' + snapshot.id) if self.configuration.quobyte_overlay_volumes: self._create_overlay_volume_from_snapshot( volume, snapshot, volume_size, out_format) else: # Copy volume from cache LOG.debug("Copying volume %(volpath)s from cache", {'volpath': path_to_new_vol}) shutil.copyfile(path_to_cached_vol, path_to_new_vol) # Note(kaisers): As writes beyond EOF are sequentialized with # FUSE we call fallocate here to optimize performance: self._fallocate_file(path_to_new_vol, volume_size) self._set_rw_permissions(path_to_new_vol)
LOG.debug('Image is raw %s', image_id) <<<<<<< HEAD self._clone_backing_file_for_volume( ======= self._clone_volume( >>>>>>> refs/remotes/openstack/stable/kilo img_file, volume['name'], volume_id=None, share=share) cloned = True break else: LOG.info( _LI('Image will locally be converted to raw %s'), image_id) dst = '%s/%s' % (dir_path, volume['name']) image_utils.convert_image(img_path, dst, 'raw', run_as_root=run_as_root) data = image_utils.qemu_img_info(dst, run_as_root=run_as_root) if data.file_format != "raw": raise exception.InvalidResults( _("Converted to raw, but" " format is now %s") % data.file_format) else: cloned = True self._register_image_in_cache( volume, image_id) break return cloned def _post_clone_image(self, volume): """Do operations post image cloning."""