def migrate_volume(self, ctxt, volume, host, thin=False, mirror_count=0): """Optimize the migration if the destination is on the same server. If the specified host is another back-end on the same server, and the volume is not attached, we can do the migration locally without going through iSCSI. """ false_ret = (False, None) if volume['status'] != 'available': return false_ret if 'location_info' not in host['capabilities']: return false_ret info = host['capabilities']['location_info'] try: (dest_type, dest_hostname, dest_vg, lvm_type, lvm_mirrors) =\ info.split(':') lvm_mirrors = int(lvm_mirrors) except ValueError: return false_ret if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname): return false_ret if dest_vg != self.vg.vg_name: vg_list = volutils.get_all_volume_groups() try: (vg for vg in vg_list if vg['name'] == dest_vg).next() except StopIteration: message = (_("Destination Volume Group %s does not exist") % dest_vg) LOG.error(message) return false_ret helper = utils.get_root_helper() dest_vg_ref = lvm.LVM(dest_vg, helper, lvm_type=lvm_type, executor=self._execute) self.remove_export(ctxt, volume) self._create_volume(volume['name'], self._sizestr(volume['size']), lvm_type, lvm_mirrors, dest_vg_ref) volutils.copy_volume(self.local_path(volume), self.local_path(volume, vg=dest_vg), volume['size'], self.configuration.volume_dd_blocksize, execute=self._execute) self._delete_volume(volume) model_update = self._create_export(ctxt, volume, vg=dest_vg) return (True, model_update) else: message = (_("Refusing to migrate volume ID: %(id)s. Please " "check your configuration because source and " "destination are the same Volume Group: %(name)s.") % {'id': volume['id'], 'name': self.vg.vg_name}) LOG.exception(message) raise exception.VolumeBackendAPIException(data=message)
def create_cloned_volume(self, volume, src_vref): """Create a cloned volume from an existing one. No cloning operation in the current release so simply copy using DD to a new space. This could be a lengthy operation. """ # Connect to source volume volname = self._get_space_name(src_vref) self._add_cinder_apphost(volname) # Make new volume provider = self.create_volume(volume) self._add_cinder_apphost(provider['provider_id']) # And copy original into it... info = self._get_space_size_redundancy(volname) volutils.copy_volume( self.local_path(src_vref), "/dev/" + provider['provider_id'], info['sizeBytes'] / units.Mi, self.configuration.volume_dd_blocksize, execute=self._execute) # That's all, folks! return provider
def test_copy_volume_dd_iflag_and_oflag(self, mock_conf, mock_exec, mock_support): fake_throttle = throttling.Throttle(["fake_throttle"]) output = volume_utils.copy_volume( "/dev/zero", "/dev/null", 1024, "3M", sync=True, execute=utils.execute, ionice=None, throttle=fake_throttle ) self.assertIsNone(output) mock_exec.assert_called_once_with( "fake_throttle", "dd", "if=/dev/zero", "of=/dev/null", "count=%s" % units.Gi, "bs=3M", "iflag=count_bytes,direct", "oflag=direct", run_as_root=True, ) mock_exec.reset_mock() output = volume_utils.copy_volume( "/dev/zero", "/dev/null", 1024, "3M", sync=False, execute=utils.execute, ionice=None, throttle=fake_throttle ) self.assertIsNone(output) mock_exec.assert_called_once_with( "fake_throttle", "dd", "if=/dev/zero", "of=/dev/null", "count=%s" % units.Gi, "bs=3M", "iflag=count_bytes,direct", "oflag=direct", run_as_root=True, )
def copy_sync_data(self, src_ldev, dest_ldev, size): src_vol = {'provider_location': six.text_type(src_ldev), 'id': 'src_vol'} dest_vol = {'provider_location': six.text_type(dest_ldev), 'id': 'dest_vol'} properties = utils.brick_get_connector_properties() driver = self.generated_from src_info = None dest_info = None try: dest_info = driver._attach_volume(self.context, dest_vol, properties) src_info = driver._attach_volume(self.context, src_vol, properties) volume_utils.copy_volume(src_info['device']['path'], dest_info['device']['path'], size * 1024, self.configuration.volume_dd_blocksize) finally: if dest_info: driver._detach_volume(self.context, dest_info, dest_vol, properties) if src_info: driver._detach_volume(self.context, src_info, src_vol, properties) self.command.discard_zero_page(dest_ldev)
def test_copy_volume_dd_no_iflag_or_oflag(self, mock_exec, mock_support): fake_throttle = throttling.Throttle(['fake_throttle']) output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, '3M', sync=True, execute=utils.execute, ionice=None, throttle=fake_throttle) self.assertIsNone(output) mock_exec.assert_called_once_with('fake_throttle', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=%s' % units.Gi, 'bs=3M', 'iflag=count_bytes', 'conv=fdatasync', run_as_root=True) mock_exec.reset_mock() output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, '3M', sync=False, execute=utils.execute, ionice=None, throttle=fake_throttle) self.assertIsNone(output) mock_exec.assert_called_once_with('fake_throttle', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=%s' % units.Gi, 'bs=3M', 'iflag=count_bytes', run_as_root=True)
def create_volume_from_snapshot(self, volume, snapshot): """Create volume from snapshot. InfiniBox does not yet support detached clone so use dd to copy data. This could be a lengthy operation. - create a clone from snapshot and map it - create a volume and map it - copy data from clone to volume - unmap volume and clone and delete the clone """ infinidat_snapshot = self._get_infinidat_snapshot(snapshot) clone_name = self._make_volume_name(volume) + '-internal' infinidat_clone = infinidat_snapshot.create_snapshot(name=clone_name) # we need a cinder-volume-like object to map the clone by name # (which is derived from the cinder id) but the clone is internal # so there is no such object. mock one clone = mock.Mock(id=str(volume.id) + '-internal') try: infinidat_volume = self._create_volume(volume) try: src_ctx = self._device_connect_context(clone) dst_ctx = self._device_connect_context(volume) with src_ctx as src_dev, dst_ctx as dst_dev: dd_block_size = self.configuration.volume_dd_blocksize vol_utils.copy_volume(src_dev['device']['path'], dst_dev['device']['path'], snapshot.volume.size * units.Ki, dd_block_size, sparse=True) except Exception: infinidat_volume.delete() raise finally: infinidat_clone.delete()
def create_volume_from_snapshot(self, volume, snapshot): """Creates a volume from a snapshot.""" self._create_volume(volume['name'], self._sizestr(volume['size'])) volutils.copy_volume(self.local_path(snapshot), self.local_path(volume), snapshot['volume_size'] * 1024, execute=self._execute)
def migrate_volume(self, ctxt, volume, host): """Optimize the migration if the destination is on the same server. If the specified host is another back-end on the same server, and the volume is not attached, we can do the migration locally without going through iSCSI. """ false_ret = (False, None) if 'location_info' not in host['capabilities']: return false_ret info = host['capabilities']['location_info'] try: (dest_type, dest_hostname, dest_vg) = info.split(':') except ValueError: return false_ret if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname): return false_ret self.remove_export(ctxt, volume) self._create_volume(volume['name'], self._sizestr(volume['size']), dest_vg) volutils.copy_volume(self.local_path(volume), self.local_path(volume, vg=dest_vg), volume['size'], execute=self._execute) self._delete_volume(volume) model_update = self._create_export(ctxt, volume, vg=dest_vg) return (True, model_update)
def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" mirror_count = 0 if self.configuration.lvm_mirrors: mirror_count = self.configuration.lvm_mirrors LOG.info(_LI('Creating clone of volume: %s'), src_vref['id']) volume_name = src_vref['name'] temp_id = 'tmp-snap-%s' % volume['id'] temp_snapshot = {'volume_name': volume_name, 'size': src_vref['size'], 'volume_size': src_vref['size'], 'name': 'clone-snap-%s' % volume['id'], 'id': temp_id} self.create_snapshot(temp_snapshot) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in try: self._create_volume(volume['name'], self._sizestr(volume['size']), self.configuration.lvm_type, mirror_count) self.vg.activate_lv(temp_snapshot['name'], is_snapshot=True) volutils.copy_volume( self.local_path(temp_snapshot), self.local_path(volume), src_vref['size'] * units.Ki, self.configuration.volume_dd_blocksize, execute=self._execute, sparse=self.sparse_copy_volume) finally: self.delete_snapshot(temp_snapshot)
def create_snapshot(self, snapshot): """Create a snapshot volume. We don't yet support snaps in SW so make a new volume and dd the source one into it. This could be a lengthy operation. """ origvol = {} origvol['name'] = snapshot['volume_name'] origvol['size'] = snapshot['volume_size'] origvol['id'] = snapshot['volume_id'] origvol['provider_id'] = snapshot.get('volume').get('provider_id') # Add me to the apphosts so I can see the volume self._add_cinder_apphost(self._get_space_name(origvol)) # Make snapshot volume snapvol = {} snapvol['display_name'] = snapshot['display_name'] snapvol['name'] = snapshot['name'] snapvol['size'] = snapshot['volume_size'] snapvol['id'] = snapshot['id'] provider = self.create_volume(snapvol) # Create_volume attaches the volume to this host, ready to snapshot. # Copy it using dd for now, we don't have real snapshots # We need to copy the entire allocated volume space, Nova will allow # full access, even beyond requested size (when our volume is larger # due to our ~1B byte alignment or cluster makeup) info = self._get_space_size_redundancy(origvol['provider_id']) volutils.copy_volume( self.local_path(origvol), "/dev/" + provider['provider_id'], info['sizeBytes'] / units.Mi, self.configuration.volume_dd_blocksize, execute=self._execute) return provider
def test_copy_volume_dd_no_iflag_or_oflag(self, mock_exec, mock_support, mock_count): fake_throttle = throttling.Throttle(["fake_throttle"]) output = volume_utils.copy_volume( "/dev/zero", "/dev/null", 1024, 1, sync=True, execute=utils.execute, ionice=None, throttle=fake_throttle ) self.assertIsNone(output) mock_exec.assert_called_once_with( "fake_throttle", "dd", "if=/dev/zero", "of=/dev/null", "count=5678", "bs=1234", "conv=fdatasync", run_as_root=True, ) mock_exec.reset_mock() output = volume_utils.copy_volume( "/dev/zero", "/dev/null", 1024, 1, sync=False, execute=utils.execute, ionice=None, throttle=fake_throttle ) self.assertIsNone(output) mock_exec.assert_called_once_with( "fake_throttle", "dd", "if=/dev/zero", "of=/dev/null", "count=5678", "bs=1234", run_as_root=True )
def create_volume_from_snapshot(self, volume, snapshot): """Creates a volume from a snapshot.""" if self.configuration.lvm_type == 'thin': self.vg.create_lv_snapshot(volume['name'], self._escape_snapshot(snapshot['name']), self.configuration.lvm_type) if volume['size'] > snapshot['volume_size']: LOG.debug("Resize the new volume to %s.", volume['size']) self.extend_volume(volume, volume['size']) # Some configurations of LVM do not automatically activate # ThinLVM snapshot LVs. self.vg.activate_lv(snapshot['name'], is_snapshot=True) self.vg.activate_lv(volume['name'], is_snapshot=True, permanent=True) return self._create_volume(volume['name'], self._sizestr(volume['size']), self.configuration.lvm_type, self.configuration.lvm_mirrors) # Some configurations of LVM do not automatically activate # ThinLVM snapshot LVs. self.vg.activate_lv(snapshot['name'], is_snapshot=True) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in volutils.copy_volume(self.local_path(snapshot), self.local_path(volume), snapshot['volume_size'] * units.Ki, self.configuration.volume_dd_blocksize, execute=self._execute, sparse=self._sparse_copy_volume)
def test_create_cloned_volume(self, _copy_volume): TEST_SRC = obj_volume.Volume(id=fake.VOLUME_ID, name_id=fake.VOLUME_NAME_ID, size=1, provider_location='/dev/loop1') TEST_VOLUME = obj_volume.Volume(name_id=fake.VOLUME2_NAME_ID, size=1, display_name='vol1') with mock.patch.object(self.drv, 'find_appropriate_size_device', return_value='/dev/loop2') as fasd_mocked: with mock.patch.object(self.drv, '_get_devices_sizes', return_value={'/dev/loop2': 2}) as \ gds_mocked: with mock.patch.object(self.drv, 'local_path', return_value='/dev/loop1') as \ lp_mocked: with mock.patch.object(self.drv, '_update_provider_location') as \ upl_mocked: volutils.copy_volume('/dev/loop1', fasd_mocked, 2, mock.sentinel, execute=self.drv._execute) self.drv.create_cloned_volume(TEST_VOLUME, TEST_SRC) fasd_mocked.assert_called_once_with(TEST_SRC.size) lp_mocked.assert_called_once_with(TEST_SRC) gds_mocked.assert_called_once_with(['/dev/loop2']) upl_mocked.assert_called_once_with( TEST_VOLUME, '/dev/loop2')
def test_create_volume_from_snapshot(self, _copy_volume): TEST_SNAP = obj_snap.Snapshot(volume_id=fake.VOLUME_ID, volume_size=1024, provider_location='/dev/loop1') TEST_VOLUME = obj_volume.Volume(id=fake.VOLUME_ID, name_id=fake.VOLUME_NAME_ID, size=1, display_name='vol1', provider_location='/dev/loop2') with mock.patch.object(self.drv, 'find_appropriate_size_device', return_value='/dev/loop2') as fasd_mocked: with mock.patch.object(self.drv, '_get_devices_sizes', return_value={'/dev/loop2': 1024}) as \ gds_mocked: with mock.patch.object(self.drv, '_update_provider_location') as \ upl_mocked: volutils.copy_volume('/dev/loop1', fasd_mocked, 1024, mock.sentinel, execute=self.drv._execute) self.drv.create_volume_from_snapshot( TEST_VOLUME, TEST_SNAP) fasd_mocked.assert_called_once_with( TEST_SNAP.volume_size) gds_mocked.assert_called_once_with(['/dev/loop2']) upl_mocked.assert_called_once_with( TEST_VOLUME, '/dev/loop2')
def delete_volume(self, volume): """ Removes the data from volume and returns the volume to pool. :param volume: OpenStack Volume Object. """ if self.configuration.sl_pool_volume_clear not in ("zero", "shred", "none"): raise exception.InvalidConfigurationValue( option="volume_clear", value=self.configuration.sl_pool_volume_clear ) sl_vol = self.meta_mgr.deserialize(volume["id"]) connection = self.vol_mgr.get_iscsi_properties(sl_vol) attach_info = self._attch(connection) size_in_mb = 1024 * volume["size"] try: if self.configuration.sl_pool_volume_clear == "zero": LOG.info("zeroing out volume") volume_utils.copy_volume("/dev/zero", attach_info["device"]["path"], size_in_mb) elif self.configuration.sl_pool_volume_clear == "shred": LOG.info("Shredding volume") utils.execute("shred", "-n3", "-s%dMiB" % size_in_mb, attach_info["device"]["path"], run_as_root=True) except proc_utils.ProcessExecutionError as ex: LOG.error(_("Error while swiping out data. %s" % ex)) raise finally: self._detach_volume(attach_info) self.meta_mgr.delete_all(volume["id"])
def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" mirror_count = 0 if self.configuration.lvm_mirrors: mirror_count = self.configuration.lvm_mirrors LOG.info(_('Creating clone of volume: %s') % src_vref['id']) volume_name = src_vref['name'] temp_id = 'tmp-snap-%s' % volume['id'] temp_snapshot = {'volume_name': volume_name, 'size': src_vref['size'], 'volume_size': src_vref['size'], 'name': 'clone-snap-%s' % volume['id'], 'id': temp_id} self.create_snapshot(temp_snapshot) self._create_volume(volume['name'], self._sizestr(volume['size']), self.configuration.lvm_type, mirror_count) self.vg.activate_lv(temp_snapshot['name'], is_snapshot=True) try: volutils.copy_volume( self.local_path(temp_snapshot), self.local_path(volume), src_vref['size'] * 1024, self.configuration.volume_dd_blocksize, execute=self._execute) finally: self.delete_snapshot(temp_snapshot)
def create_cloned_volume(self, volume, src_vref): """Create a clone from source volume. InfiniBox does not yet support detached clone so use dd to copy data. This could be a lengthy operation. * map source volume * create and map new volume * copy data from source to new volume * unmap both volumes """ self._asssert_volume_not_mapped(src_vref) infinidat_volume = self._create_volume(volume) try: src_ctx = self._device_connect_context(src_vref) dst_ctx = self._device_connect_context(volume) with src_ctx as src_dev, dst_ctx as dst_dev: dd_block_size = self.configuration.volume_dd_blocksize vol_utils.copy_volume(src_dev['device']['path'], dst_dev['device']['path'], src_vref.size * units.Ki, dd_block_size, sparse=True) except Exception: infinidat_volume.delete() raise
def create_volume_from_snapshot(self, volume, snapshot): """Create volume from snapshot. - search for snapshot and retention_policy - create a view from snapshot and attach view - create a volume and attach volume - copy data from attached view to attached volume - detach volume and view and finally delete view """ snap_name = self.get_snap_name(snapshot.id) view_name = self.get_view_name(volume.id) vol_name = self.get_volume_name(volume.id) cview = src_attach_info = dest_attach_info = None rpolicy = self.get_policy() properties = utils.brick_get_connector_properties() LOG.debug("Searching for snapshot: %s in K2.", snap_name) snap_rs = self.client.search("snapshots", short_name=snap_name) if hasattr(snap_rs, 'hits') and snap_rs.total != 0: snap = snap_rs.hits[0] LOG.debug("Creating a view: %(view)s from snapshot: %(snap)s", {'view': view_name, 'snap': snap_name}) try: cview = self.client.new("snapshots", short_name=view_name, source=snap, retention_policy=rpolicy, is_exposable=True).save() except Exception as ex: LOG.exception(_LE("Creating a view: %(view)s from snapshot: " "%(snap)s failed"), {"view": view_name, "snap": snap_name}) raise exception.KaminarioCinderDriverException( reason=six.text_type(ex.message)) else: msg = _("Snapshot: %s search failed in K2.") % snap_name LOG.error(msg) raise exception.KaminarioCinderDriverException(reason=msg) try: conn = self.initialize_connection(cview, properties) src_attach_info = self._connect_device(conn) self.create_volume(volume) conn = self.initialize_connection(volume, properties) dest_attach_info = self._connect_device(conn) vol_utils.copy_volume(src_attach_info['device']['path'], dest_attach_info['device']['path'], snapshot.volume.size * units.Ki, self.configuration.volume_dd_blocksize, sparse=True) self.terminate_connection(volume, properties) self.terminate_connection(cview, properties) except Exception as ex: self.terminate_connection(cview, properties) self.terminate_connection(volume, properties) cview.delete() self.delete_volume(volume) LOG.exception(_LE("Copy to volume: %(vol)s from view: %(view)s " "failed"), {"vol": vol_name, "view": view_name}) raise exception.KaminarioCinderDriverException( reason=six.text_type(ex.message))
def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" mirror_count = 0 if self.configuration.lvm_mirrors: mirror_count = self.configuration.lvm_mirrors LOG.info(_("Creating clone of volume: %s") % src_vref["id"]) volume_name = src_vref["name"] temp_id = "tmp-snap-%s" % volume["id"] temp_snapshot = { "volume_name": volume_name, "size": src_vref["size"], "volume_size": src_vref["size"], "name": "clone-snap-%s" % volume["id"], "id": temp_id, } self.create_snapshot(temp_snapshot) self._create_volume(volume["name"], self._sizestr(volume["size"]), self.configuration.lvm_type, mirror_count) self.vg.activate_lv(temp_snapshot["name"], is_snapshot=True) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in try: volutils.copy_volume( self.local_path(temp_snapshot), self.local_path(volume), src_vref["size"] * units.KiB, self.configuration.volume_dd_blocksize, execute=self._execute, ) finally: self.delete_snapshot(temp_snapshot)
def test_clear_volume_zero(self): CONF.volume_clear = 'zero' CONF.volume_clear_size = 1 self.mox.StubOutWithMock(volume_utils, 'copy_volume') volume_utils.copy_volume("/dev/zero", "volume_path", 1, CONF.volume_dd_blocksize, sync=True, execute=utils.execute) self.mox.ReplayAll() volume_utils.clear_volume(1024, "volume_path")
def copy_volume_data(self, context, src_vol, dest_vol, remote=None): """Copy data from src_vol to dest_vol.""" LOG.debug(_('copy_data_between_volumes %(src)s -> %(dest)s.') % {'src': src_vol['name'], 'dest': dest_vol['name']}) properties = utils.brick_get_connector_properties() dest_remote = True if remote in ['dest', 'both'] else False dest_orig_status = dest_vol['status'] try: dest_attach_info = self._attach_volume(context, dest_vol, properties, remote=dest_remote) except Exception: with excutils.save_and_reraise_exception(): msg = _("Failed to attach volume %(vol)s") LOG.error(msg % {'vol': dest_vol['id']}) self.db.volume_update(context, dest_vol['id'], {'status': dest_orig_status}) src_remote = True if remote in ['src', 'both'] else False src_orig_status = src_vol['status'] try: src_attach_info = self._attach_volume(context, src_vol, properties, remote=src_remote) except Exception: with excutils.save_and_reraise_exception(): msg = _("Failed to attach volume %(vol)s") LOG.error(msg % {'vol': src_vol['id']}) self.db.volume_update(context, src_vol['id'], {'status': src_orig_status}) self._copy_volume_data_cleanup(context, dest_vol, properties, dest_attach_info, dest_remote, force=True) try: size_in_mb = int(src_vol['size']) * 1024 # vol size is in GB volume_utils.copy_volume( src_attach_info['device']['path'], dest_attach_info['device']['path'], size_in_mb, self.configuration.volume_dd_blocksize) copy_error = False except Exception: with excutils.save_and_reraise_exception(): msg = _("Failed to copy volume %(src)s to %(dest)d") LOG.error(msg % {'src': src_vol['id'], 'dest': dest_vol['id']}) copy_error = True finally: self._copy_volume_data_cleanup(context, dest_vol, properties, dest_attach_info, dest_remote, force=copy_error) self._copy_volume_data_cleanup(context, src_vol, properties, src_attach_info, src_remote, force=copy_error)
def create_cloned_volume(self, volume, src_vref): LOG.info(_('Creating clone of volume: %s') % src_vref['id']) device = self.find_appropriate_size_device(src_vref['size']) volutils.copy_volume(self.local_path(src_vref), device, self._get_device_size(device) * 2048, execute=self._execute) return { 'provider_location': self._iscsi_location(None, None, None, None, device), }
def test_clear_volume_zero(self): CONF.volume_clear = "zero" CONF.volume_clear_size = 1 CONF.volume_clear_ionice = None self.mox.StubOutWithMock(volume_utils, "copy_volume") volume_utils.copy_volume( "/dev/zero", "volume_path", 1, CONF.volume_dd_blocksize, sync=True, ionice=None, execute=utils.execute ) self.mox.ReplayAll() volume_utils.clear_volume(1024, "volume_path")
def create_volume_from_snapshot(self, volume, snapshot): LOG.info(_LI('Creating volume %s from snapshot.'), volume.id) device = self.find_appropriate_size_device(snapshot.volume_size) dev_size = self._get_devices_sizes([device]) volutils.copy_volume( self.local_path(snapshot), device, dev_size[device], self.configuration.volume_dd_blocksize, execute=self._execute) self._update_provider_location(volume, device)
def create_cloned_volume(self, volume, src_vref): LOG.info(_LI('Creating clone of volume: %s.'), src_vref.id) device = self.find_appropriate_size_device(src_vref.size) dev_size = self._get_devices_sizes([device]) volutils.copy_volume( self.local_path(src_vref), device, dev_size[device], self.configuration.volume_dd_blocksize, execute=self._execute) self._update_provider_location(volume, device)
def migrate_volume(self, ctxt, volume, host, thin=False, mirror_count=0): """Optimize the migration if the destination is on the same server. If the specified host is another back-end on the same server, and the volume is not attached, we can do the migration locally without going through iSCSI. """ false_ret = (False, None) if volume['status'] != 'available': return false_ret if 'location_info' not in host['capabilities']: return false_ret info = host['capabilities']['location_info'] try: (dest_type, dest_hostname, dest_vg, lvm_type, lvm_mirrors) =\ info.split(':') lvm_mirrors = int(lvm_mirrors) except ValueError: return false_ret if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname): return false_ret if dest_vg != self.vg.vg_name: vg_list = volutils.get_all_volume_groups() try: (vg for vg in vg_list if vg['name'] == dest_vg).next() except StopIteration: message = (_("Destination Volume Group %s does not exist") % dest_vg) LOG.error(message) return false_ret helper = utils.get_root_helper() dest_vg_ref = lvm.LVM(dest_vg, helper, lvm_type=lvm_type, executor=self._execute) self.remove_export(ctxt, volume) self._create_volume(volume['name'], self._sizestr(volume['size']), lvm_type, lvm_mirrors, dest_vg_ref) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in size_in_mb = volume['size'] * units.Ki volutils.copy_volume(self.local_path(volume), self.local_path(volume, vg=dest_vg), size_in_mb, self.configuration.volume_dd_blocksize, execute=self._execute) self._delete_volume(volume) model_update = self._create_export(ctxt, volume, vg=dest_vg) return (True, model_update)
def create_cloned_volume(self, volume, src_vref): LOG.info(_LI('Creating clone of volume: %s'), src_vref['id']) device = self.find_appropriate_size_device(src_vref['size']) volutils.copy_volume( self.local_path(src_vref), device, self._get_device_size(device) * 2048, self.configuration.volume_dd_blocksize, execute=self._execute) return { 'provider_location': device, }
def create_cloned_volume(self, volume, src_vref): LOG.info(_("Creating clone of volume: %s") % src_vref["id"]) device = self.find_appropriate_size_device(src_vref["size"]) volutils.copy_volume( self.local_path(src_vref), device, self._get_device_size(device) * 2048, self.configuration.volume_dd_blocksize, execute=self._execute, ) return {"provider_location": device}
def _copy_vdisk_data(self, src_vdisk_name, src_vdisk_id, dest_vdisk_name, dest_vdisk_id): """Copy data from src vdisk to dest vdisk. To be able to copy data between vdisks, we must ensure that both vdisks have been mapped to host. If vdisk has not been mapped, it must be mapped firstly. When data copy completed, vdisk should be restored to previous mapped or non-mapped status. """ LOG.debug("enter: _copy_vdisk_data: %(src)s -> %(dest)s.", {"src": src_vdisk_name, "dest": dest_vdisk_name}) connector = utils.brick_get_connector_properties() (src_map, src_lun_id) = self._is_vdisk_map(src_vdisk_name, connector) (dest_map, dest_lun_id) = self._is_vdisk_map(dest_vdisk_name, connector) src_map_device = None src_properties = None dest_map_device = None dest_properties = None try: if not src_map: src_lun_id = self._map_vdisk_to_host(src_vdisk_name, connector) if not dest_map: dest_lun_id = self._map_vdisk_to_host(dest_vdisk_name, connector) src_properties = self._get_vdisk_map_properties( connector, src_lun_id, src_vdisk_name, src_vdisk_id, self._get_vdisk_params(None) ) src_map_device = self._scan_device(src_properties) dest_properties = self._get_vdisk_map_properties( connector, dest_lun_id, dest_vdisk_name, dest_vdisk_id, self._get_vdisk_params(None) ) dest_map_device = self._scan_device(dest_properties) src_vdisk_attr = self._get_vdisk_attributes(src_vdisk_name) # vdisk capacity is bytes, translate into MB size_in_mb = int(src_vdisk_attr["capacity"]) / units.Mi volume_utils.copy_volume( src_map_device["path"], dest_map_device["path"], size_in_mb, self.configuration.volume_dd_blocksize ) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Failed to copy %(src)s to %(dest)s."), {"src": src_vdisk_name, "dest": dest_vdisk_name}) finally: if not dest_map: self._unmap_vdisk_from_host(dest_vdisk_name, connector) self._remove_device(dest_properties, dest_map_device) if not src_map: self._unmap_vdisk_from_host(src_vdisk_name, connector) self._remove_device(src_properties, src_map_device) LOG.debug("leave: _copy_vdisk_data: %(src)s -> %(dest)s.", {"src": src_vdisk_name, "dest": dest_vdisk_name})
def test_copy_volume_dd_iflag_and_oflag(self): def fake_utils_execute(*cmd, **kwargs): if 'if=/dev/zero' in cmd and 'iflag=direct' in cmd: raise processutils.ProcessExecutionError() if 'of=/dev/null' in cmd and 'oflag=direct' in cmd: raise processutils.ProcessExecutionError() if 'iflag=direct' in cmd and 'oflag=direct' in cmd: raise exception.InvalidInput(message='iflag/oflag error') volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, CONF.volume_dd_blocksize, sync=True, ionice=None, execute=fake_utils_execute)
def _dd_copy(self, vol_params, src_snap, src_lun=None): """Creates a volume via copying a Unity snapshot. It attaches the `volume` and `snap`, then use `dd` to copy the data from the Unity snapshot to the `volume`. """ dest_lun = self.client.create_lun( name=vol_params.name, size=vol_params.size, pool=vol_params.pool, description=vol_params.description, io_limit_policy=vol_params.io_limit_policy) src_id = src_snap.get_id() try: conn_props = cinder_utils.brick_get_connector_properties() with self._connect_resource(dest_lun, conn_props, vol_params.volume_id) as dest_info, \ self._connect_resource(src_snap, conn_props, src_id) as src_info: if src_lun is None: # If size is not specified, need to get the size from LUN # of snapshot. lun = self.client.get_lun( lun_id=src_snap.storage_resource.get_id()) size_in_m = utils.byte_to_mib(lun.size_total) else: size_in_m = utils.byte_to_mib(src_lun.size_total) vol_utils.copy_volume( src_info['device']['path'], dest_info['device']['path'], size_in_m, self.driver.configuration.volume_dd_blocksize, sparse=True) except Exception: with excutils.save_and_reraise_exception(): utils.ignore_exception(self.client.delete_lun, dest_lun.get_id()) LOG.error('Failed to create cloned volume: %(vol_id)s, ' 'from source unity snapshot: %(snap_name)s.', {'vol_id': vol_params.volume_id, 'snap_name': src_snap.name}) return dest_lun
def test_copy_volume_dd_iflag_and_oflag(self, mock_conf, mock_exec, mock_support, mock_count): fake_throttle = throttling.Throttle(['fake_throttle']) output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=True, execute=utils.execute, ionice=None, throttle=fake_throttle) self.assertIsNone(output) mock_exec.assert_called_once_with('fake_throttle', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'iflag=direct', 'oflag=direct', run_as_root=True) mock_exec.reset_mock() output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=False, execute=utils.execute, ionice=None, throttle=fake_throttle) self.assertIsNone(output) mock_exec.assert_called_once_with('fake_throttle', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'iflag=direct', 'oflag=direct', run_as_root=True)
def test_copy_volume_dd_with_sparse_iflag_and_oflag(self, mock_exec, mock_support, mock_count): output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=True, execute=utils.execute, sparse=True) self.assertIsNone(output) mock_exec.assert_called_once_with('dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'iflag=direct', 'oflag=direct', 'conv=sparse', run_as_root=True)
def create_volume_from_snapshot(self, volume, snapshot): """Creates a volume from a snapshot.""" self._create_volume(volume['name'], self._sizestr(volume['size']), self.configuration.lvm_type, self.configuration.lvm_mirrors) # Terrifying patch to ignore snapshoting when migrating from ceph volume_type_id if volume['volume_type_id'] != 'b07b4b6d-ac74-4d0b-9e68-cc1a637798b0': # Some configurations of LVM do not automatically activate # ThinLVM snapshot LVs. self.vg.activate_lv(snapshot['name'], is_snapshot=True) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in volutils.copy_volume(self.local_path(snapshot), self.local_path(volume), snapshot['volume_size'] * units.Ki, self.configuration.volume_dd_blocksize, execute=self._execute, sparse=self.sparse_copy_volume)
def _copy_with_dd(self, src_ldev, dest_ldev, size): """Copy the content of a volume by dd command.""" src_info = None dest_info = None properties = cinder_utils.brick_get_connector_properties( multipath=self.conf.use_multipath_for_image_xfer, enforce_multipath=self.conf.enforce_multipath_for_image_xfer) try: dest_info = self._attach_ldev(dest_ldev, properties) src_info = self._attach_ldev(src_ldev, properties) volume_utils.copy_volume(src_info['device']['path'], dest_info['device']['path'], size * units.Ki, self.conf.volume_dd_blocksize) finally: if src_info: self._detach_ldev(src_info, src_ldev, properties) if dest_info: self._detach_ldev(dest_info, dest_ldev, properties) self.discard_zero_page({'provider_location': six.text_type(dest_ldev)})
def _create_volume_from_snap(self, volume, snap, size_in_m=None): """Creates a volume from a Unity snapshot. It attaches the `volume` and `snap`, then use `dd` to copy the data from the Unity snapshot to the `volume`. """ model_update = self.create_volume(volume) # Update `provider_location` and `provider_id` of `volume` explicitly. volume.update(model_update) src_id = snap.get_id() dest_lun = self.client.get_lun(lun_id=self.get_lun_id(volume)) try: conn_props = cinder_utils.brick_get_connector_properties() with self._connect_resource(dest_lun, conn_props, volume.id) as dest_info, \ self._connect_resource(snap, conn_props, src_id) as src_info: if size_in_m is None: # If size is not specified, need to get the size from LUN # of snapshot. lun = self.client.get_lun( lun_id=snap.storage_resource.get_id()) size_in_m = utils.byte_to_mib(lun.size_total) vol_utils.copy_volume( src_info['device']['path'], dest_info['device']['path'], size_in_m, self.driver.configuration.volume_dd_blocksize, sparse=True) except Exception: with excutils.save_and_reraise_exception(): utils.ignore_exception(self.delete_volume, volume) LOG.error( 'Failed to create cloned volume: %(vol_id)s, ' 'from source unity snapshot: %(snap_name)s.', { 'vol_id': volume.id, 'snap_name': snap.name }) return model_update
def clear_volume(self, volume, is_snapshot=False): """unprovision old volumes to prevent data leaking between users.""" # NOTE(jdg): Don't write the blocks of thin provisioned # volumes if self.configuration.volume_clear == 'none' or \ self.configuration.lvm_type == 'thin': return if is_snapshot: # if the volume to be cleared is a snapshot of another volume # we need to clear out the volume using the -cow instead of the # directly volume path. We need to skip this if we are using # thin provisioned LVs. # bug# lp1191812 dev_path = self.local_path(volume) + "-cow" else: dev_path = self.local_path(volume) if not os.path.exists(dev_path): msg = (_('Volume device file path %s does not exist.') % dev_path) LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) size_in_g = volume.get('size', volume.get('volume_size', None)) if size_in_g is None: msg = (_("Size for volume: %s not found, " "cannot secure delete.") % volume['id']) LOG.error(msg) raise exception.InvalidParameterValue(msg) size_in_m = self.configuration.volume_clear_size LOG.info(_("Performing secure delete on volume: %s") % volume['id']) if self.configuration.volume_clear == 'zero': if size_in_m == 0: return volutils.copy_volume('/dev/zero', dev_path, size_in_g * 1024, sync=True, execute=self._execute) else: clear_cmd = ['shred', '-n0', '-z', '-s%dMiB' % size_in_m] elif self.configuration.volume_clear == 'shred': clear_cmd = ['shred', '-n3'] if size_in_m: clear_cmd.append('-s%dMiB' % size_in_m) else: raise exception.InvalidConfigurationValue( option='volume_clear', value=self.configuration.volume_clear) clear_cmd.append(dev_path) self._execute(*clear_cmd, run_as_root=True)
def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" LOG.info(_('Creating clone of volume: %s') % src_vref['id']) volume_name = src_vref['name'] temp_id = 'tmp-snap-%s' % volume['id'] temp_snapshot = { 'volume_name': volume_name, 'size': src_vref['size'], 'volume_size': src_vref['size'], 'name': 'clone-snap-%s' % volume['id'], 'id': temp_id } self.create_snapshot(temp_snapshot) self._create_volume(volume['name'], self._sizestr(volume['size'])) try: volutils.copy_volume(self.local_path(temp_snapshot), self.local_path(volume), src_vref['size'] * 1024, execute=self._execute) finally: self.delete_snapshot(temp_snapshot)
def test_copy_volume_dd_iflag_and_oflag(self, mock_conf, mock_exec, mock_support, mock_count, mock_cg): mock_conf.volume_copy_bps_limit = 10 output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=True, execute=utils.execute, ionice=None) self.assertIsNone(output) mock_exec.assert_called_once_with('cg_cmd', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'iflag=direct', 'oflag=direct', run_as_root=True) mock_exec.reset_mock() output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=False, execute=utils.execute, ionice=None) self.assertIsNone(output) mock_exec.assert_called_once_with('cg_cmd', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'iflag=direct', 'oflag=direct', run_as_root=True)
def create_cloned_volume(self, volume, src_vref): """Create a clone from source volume. - attach source volume - create and attach new volume - copy data from attached source volume to attached new volume - detach both volumes """ clone_name = self.get_volume_name(volume.id) src_name = self.get_volume_name(src_vref.id) src_vol = self.client.search("volumes", name=src_name) src_map = self.client.search("mappings", volume=src_vol) if src_map.total != 0: msg = _("K2 driver does not support clone of a attached volume. " "To get this done, create a snapshot from the attached " "volume and then create a volume from the snapshot.") LOG.error(msg) raise exception.KaminarioCinderDriverException(reason=msg) try: properties = utils.brick_get_connector_properties() conn = self.initialize_connection(src_vref, properties) src_attach_info = self._connect_device(conn) self.create_volume(volume) conn = self.initialize_connection(volume, properties) dest_attach_info = self._connect_device(conn) vol_utils.copy_volume(src_attach_info['device']['path'], dest_attach_info['device']['path'], src_vref.size * units.Ki, self.configuration.volume_dd_blocksize, sparse=True) self.terminate_connection(volume, properties) self.terminate_connection(src_vref, properties) except Exception as ex: self.terminate_connection(src_vref, properties) self.terminate_connection(volume, properties) self.delete_volume(volume) LOG.exception(_LE("Create a clone: %s failed."), clone_name) raise exception.KaminarioCinderDriverException( reason=six.text_type(ex.message))
def test_copy_volume_dd_iflag_and_oflag(self): def fake_utils_execute(*cmd, **kwargs): if 'if=/dev/zero' in cmd and 'iflag=direct' in cmd: raise processutils.ProcessExecutionError() if 'of=/dev/null' in cmd and 'oflag=direct' in cmd: raise processutils.ProcessExecutionError() if 'iflag=direct' in cmd and 'oflag=direct' in cmd: raise exception.InvalidInput(message='iflag/oflag error') def fake_check_odirect(src, dest, flags='blah'): return False self.stubs.Set(volume_utils, 'check_for_odirect_support', fake_check_odirect) volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, CONF.volume_dd_blocksize, sync=True, ionice=None, execute=fake_utils_execute)
def test_create_cloned_volume(self, _copy_volume): TEST_SRC = {'id': '1', 'size': 1, 'provider_location': '1 2 3 /dev/loop1'} TEST_VOLUME = {} with mock.patch.object(self.drv, 'find_appropriate_size_device', return_value='/dev/loop2') as fasd_mocked: with mock.patch.object(self.drv, '_get_device_size', return_value=1) as gds_mocked: with mock.patch.object(self.drv, 'local_path', return_value='/dev/loop1') as \ lp_mocked: volutils.copy_volume('/dev/loop1', fasd_mocked, 2048, mock.sentinel, execute=self.drv._execute) self.assertEqual(self.drv.create_cloned_volume( TEST_VOLUME, TEST_SRC), {'provider_location': '/dev/loop2'}) fasd_mocked.assert_called_once_with(TEST_SRC['size']) lp_mocked.assert_called_once_with(TEST_SRC) gds_mocked.assert_called_once_with('/dev/loop2')
def _create_and_copy_volume(self, dstvol, srcvol): """Creates a volume from a volume or a snapshot.""" updates = self._create_file(dstvol) # We need devices attached for IO operations. with temp_lvm_device(self, srcvol) as vg, \ temp_raw_device(self, dstvol): self._setup_lvm(dstvol) # Some configurations of LVM do not automatically activate # ThinLVM snapshot LVs. with patched(lvm.LVM, 'activate_lv', self._activate_lv): vg.activate_lv(srcvol['name'], True) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in volutils.copy_volume(self._mapper_path(srcvol), self._mapper_path(dstvol), srcvol['volume_size'] * units.Ki, self.configuration.volume_dd_blocksize, execute=self._execute) return updates
def test_create_cloned_volume(self): TEST_SRC = { 'id': '1', 'size': 1, 'provider_location': '1 2 3 /dev/loop1' } TEST_VOLUME = {} self.mox.StubOutWithMock(self.drv, 'find_appropriate_size_device') dev = self.drv.find_appropriate_size_device(TEST_SRC['size']).\ AndReturn('/dev/loop2') self.mox.StubOutWithMock(volutils, 'copy_volume') self.mox.StubOutWithMock(self.drv, 'local_path') self.mox.StubOutWithMock(self.drv, '_get_device_size') self.drv.local_path(TEST_SRC).AndReturn('/dev/loop1') self.drv._get_device_size('/dev/loop2').AndReturn(1) volutils.copy_volume('/dev/loop1', dev, 2048, mox.IgnoreArg(), execute=self.drv._execute) self.mox.ReplayAll() self.assertEqual(self.drv.create_cloned_volume(TEST_VOLUME, TEST_SRC), {'provider_location': '/dev/loop2'})
def create_cloned_volume(self, volume, src_vref): """Creates a clone of the specified volume.""" mirror_count = 0 if self.configuration.lvm_mirrors: mirror_count = self.configuration.lvm_mirrors LOG.info(_LI('Creating clone of volume: %s'), src_vref['id']) volume_name = src_vref['name'] temp_id = 'tmp-snap-%s' % volume['id'] temp_snapshot = {'volume_name': volume_name, 'size': src_vref['size'], 'volume_size': src_vref['size'], 'name': 'clone-snap-%s' % volume['id'], 'id': temp_id} self.create_snapshot(temp_snapshot) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in try: self._create_volume(volume['name'], self._sizestr(volume['size']), self.configuration.lvm_type, mirror_count) self.vg.activate_lv(temp_snapshot['name'], is_snapshot=True) sparse = True if self.configuration.lvm_type == 'thin' else False volutils.copy_volume( self.local_path(temp_snapshot), self.local_path(volume), src_vref['size'] * units.Ki, self.configuration.volume_dd_blocksize, execute=self._execute, sparse=sparse) finally: self.delete_snapshot(temp_snapshot)
def create_volume_from_snapshot(self, volume, snapshot): """Create volume from snapshot. InfiniBox does not yet support detached clone so use dd to copy data. This could be a lengthy operation. - create a clone from snapshot and map it - create a volume and map it - copy data from clone to volume - unmap volume and clone and delete the clone """ snapshot_id = self._get_infinidat_snapshot_id(snapshot) clone_name = self._make_volume_name(volume) + '-internal' infinidat_clone = self._post( 'volumes', dict(parent_id=snapshot_id, name=clone_name)) # we need a cinder-volume-like object to map the clone by name # (which is derived from the cinder id) but the clone is internal # so there is no such object. mock one clone = mock.Mock(id=str(volume.id) + '-internal') try: infinidat_volume = self._create_volume(volume) try: src_ctx = self._device_connect_context(clone) dst_ctx = self._device_connect_context(volume) with src_ctx as src_dev, dst_ctx as dst_dev: dd_block_size = self.configuration.volume_dd_blocksize vol_utils.copy_volume(src_dev['device']['path'], dst_dev['device']['path'], snapshot.volume.size * units.Ki, dd_block_size, sparse=True) except Exception: self._delete(DELETE_URI % infinidat_volume['id']) raise finally: self._delete(DELETE_URI % infinidat_clone['id'])
def create_cloned_volume(self, volume, src_vref): """Create a cloned volume from an existing one. No cloning operation in the current release so simply copy using DD to a new space. This could be a lengthy operation. """ # Connect to source volume volname = self._get_space_name(src_vref) self._add_cinder_apphost(volname) # Make new volume provider = self.create_volume(volume) self._add_cinder_apphost(provider['provider_id']) # And copy original into it... info = self._get_space_size_redundancy(volname) volutils.copy_volume(self.local_path(src_vref), "/dev/" + provider['provider_id'], info['sizeBytes'] // units.Mi, self.configuration.volume_dd_blocksize, execute=self._execute) # That's all, folks! return provider
def test_copy_volume_dd_no_throttle(self, mock_exec, mock_support, mock_count): output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=True, execute=utils.execute, ionice=None) self.assertIsNone(output) mock_exec.assert_called_once_with('dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'conv=fdatasync', run_as_root=True)
def test_copy_volume_dd_with_sparse(self, mock_exec, mock_support): output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, '3M', sync=True, execute=utils.execute, sparse=True) self.assertIsNone(output) mock_exec.assert_called_once_with('dd', 'if=/dev/zero', 'of=/dev/null', 'count=%s' % units.Gi, 'bs=3M', 'iflag=count_bytes', 'conv=fdatasync,sparse', run_as_root=True)
def test_copy_volume_dd_with_ionice(self, mock_conf, mock_exec, mock_support, mock_count, mock_cg): mock_conf.volume_copy_bps_limit = 10 output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1, sync=True, execute=utils.execute, ionice='-c3') self.assertIsNone(output) mock_exec.assert_called_once_with('ionice', '-c3', 'dd', 'if=/dev/zero', 'of=/dev/null', 'count=5678', 'bs=1234', 'conv=fdatasync', run_as_root=True)
def clear_volume(self, volume): """unprovision old volumes to prevent data leaking between users.""" if self.configuration.volume_clear == 'none': return vol_path = self.local_path(volume) size_in_g = volume.get('size', volume.get('volume_size', None)) if size_in_g is None: LOG.warning( _("Size for volume: %s not found, " "skipping secure delete.") % volume['id']) return size_in_m = self.configuration.volume_clear_size LOG.info(_("Performing secure delete on volume: %s") % volume['id']) if self.configuration.volume_clear == 'zero': if size_in_m == 0: return volutils.copy_volume('/dev/zero', vol_path, size_in_g * 1024, sync=True, execute=self._execute) else: clear_cmd = ['shred', '-n0', '-z', '-s%dMiB' % size_in_m] elif self.configuration.volume_clear == 'shred': clear_cmd = ['shred', '-n3'] if size_in_m: clear_cmd.append('-s%dMiB' % size_in_m) else: LOG.error(_("Error unrecognized volume_clear option: %s"), self.configuration.volume_clear) return clear_cmd.append(vol_path) self._execute(*clear_cmd, run_as_root=True)
def copy_volume_data(self, context, src_vol, dest_vol, remote=None): """Copy data from src_vol to dest_vol.""" LOG.debug( _('copy_data_between_volumes %(src)s -> %(dest)s.') % { 'src': src_vol['name'], 'dest': dest_vol['name'] }) properties = utils.brick_get_connector_properties() dest_remote = True if remote in ['dest', 'both'] else False dest_orig_status = dest_vol['status'] try: dest_attach_info = self._attach_volume(context, dest_vol, properties, remote=dest_remote) except Exception: with excutils.save_and_reraise_exception(): msg = _("Failed to attach volume %(vol)s") LOG.error(msg % {'vol': dest_vol['id']}) self.db.volume_update(context, dest_vol['id'], {'status': dest_orig_status}) src_remote = True if remote in ['src', 'both'] else False src_orig_status = src_vol['status'] try: src_attach_info = self._attach_volume(context, src_vol, properties, remote=src_remote) except Exception: with excutils.save_and_reraise_exception(): msg = _("Failed to attach volume %(vol)s") LOG.error(msg % {'vol': src_vol['id']}) self.db.volume_update(context, src_vol['id'], {'status': src_orig_status}) self._detach_volume(context, dest_attach_info, dest_vol, properties, force=True, remote=dest_remote) try: size_in_mb = int(src_vol['size']) * 1024 # vol size is in GB volume_utils.copy_volume(src_attach_info['device']['path'], dest_attach_info['device']['path'], size_in_mb, self.configuration.volume_dd_blocksize) copy_error = False except Exception: with excutils.save_and_reraise_exception(): msg = _("Failed to copy volume %(src)s to %(dest)d") LOG.error(msg % {'src': src_vol['id'], 'dest': dest_vol['id']}) copy_error = True finally: self._detach_volume(context, dest_attach_info, dest_vol, properties, force=copy_error, remote=dest_remote) self._detach_volume(context, src_attach_info, src_vol, properties, force=copy_error, remote=src_remote)
def test_copy_volume_handle_transfer(self, mock_open, mock_transfer): handle = io.RawIOBase() output = volume_utils.copy_volume('/foo/bar', handle, 1024, 1) self.assertIsNone(output) mock_transfer.assert_called_once_with(mock.ANY, mock.ANY, 1073741824, mock.ANY)
def test_copy_volume_handles(self, mock_copy): handle1 = io.RawIOBase() handle2 = io.RawIOBase() output = volume_utils.copy_volume(handle1, handle2, 1024, 1) self.assertIsNone(output) mock_copy.assert_called_once_with(handle1, handle2, 1024)
def fetch_to_volume_format(context, image_service, image_id, dest, volume_format, blocksize, user_id=None, project_id=None, size=None, run_as_root=True): qemu_img = True image_meta = image_service.show(context, image_id) # NOTE(avishay): I'm not crazy about creating temp files which may be # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because # it seeks. Maybe we can think of something for a future version. with temporary_file() as tmp: has_meta = False if not image_meta else True try: format_raw = True if image_meta['disk_format'] == 'raw' else False except TypeError: format_raw = False data = get_qemu_data(image_id, has_meta, format_raw, tmp, run_as_root) if data is None: qemu_img = False tmp_images = TemporaryImages.for_image_service(image_service) tmp_image = tmp_images.get(context, image_id) if tmp_image: tmp = tmp_image else: fetch(context, image_service, image_id, tmp, user_id, project_id) if is_xenserver_format(image_meta): replace_xenserver_image_with_coalesced_vhd(tmp) if not qemu_img: # qemu-img is not installed but we do have a RAW image. As a # result we only need to copy the image to the destination and then # return. LOG.debug('Copying image from %(tmp)s to volume %(dest)s - ' 'size: %(size)s', {'tmp': tmp, 'dest': dest, 'size': image_meta['size']}) image_size_m = math.ceil(float(image_meta['size']) / units.Mi) volume_utils.copy_volume(tmp, dest, image_size_m, blocksize) return data = qemu_img_info(tmp, run_as_root=run_as_root) virt_size = int(math.ceil(float(data.virtual_size) / units.Gi)) # NOTE(xqueralt): If the image virtual size doesn't fit in the # requested volume there is no point on resizing it because it will # generate an unusable image. if size is not None and virt_size > size: params = {'image_size': virt_size, 'volume_size': size} reason = _("Size is %(image_size)dGB and doesn't fit in a " "volume of size %(volume_size)dGB.") % params raise exception.ImageUnacceptable(image_id=image_id, reason=reason) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file, }) # NOTE(e0ne): check for free space in destination directory before # image convertion. check_available_space(dest, virt_size, image_id) # NOTE(jdg): I'm using qemu-img convert to write # to the volume regardless if it *needs* conversion or not # TODO(avishay): We can speed this up by checking if the image is raw # and if so, writing directly to the device. However, we need to keep # check via 'qemu-img info' that what we copied was in fact a raw # image and not a different format with a backing file, which may be # malicious. LOG.debug("%s was %s, converting to %s ", image_id, fmt, volume_format) disk_format = fixup_disk_format(image_meta['disk_format']) convert_image(tmp, dest, volume_format, src_format=disk_format, run_as_root=run_as_root)
def migrate_volume(self, ctxt, volume, host, thin=False, mirror_count=0): """Optimize the migration if the destination is on the same server. If the specified host is another back-end on the same server, and the volume is not attached, we can do the migration locally without going through iSCSI. """ false_ret = (False, None) if volume['status'] != 'available': return false_ret if 'location_info' not in host['capabilities']: return false_ret info = host['capabilities']['location_info'] try: (dest_type, dest_hostname, dest_vg, lvm_type, lvm_mirrors) =\ info.split(':') lvm_mirrors = int(lvm_mirrors) except ValueError: return false_ret if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname): return false_ret if dest_vg == self.vg.vg_name: message = (_("Refusing to migrate volume ID: %(id)s. Please " "check your configuration because source and " "destination are the same Volume Group: %(name)s.") % { 'id': volume['id'], 'name': self.vg.vg_name }) LOG.error(message) raise exception.VolumeBackendAPIException(data=message) vg_list = volutils.get_all_volume_groups() try: next(vg for vg in vg_list if vg['name'] == dest_vg) except StopIteration: LOG.error(_LE("Destination Volume Group %s does not exist"), dest_vg) return false_ret helper = utils.get_root_helper() lvm_conf_file = self.configuration.lvm_conf_file if lvm_conf_file.lower() == 'none': lvm_conf_file = None dest_vg_ref = lvm.LVM(dest_vg, helper, lvm_type=lvm_type, executor=self._execute, lvm_conf=lvm_conf_file) self._create_volume(volume['name'], self._sizestr(volume['size']), lvm_type, lvm_mirrors, dest_vg_ref) # copy_volume expects sizes in MiB, we store integer GiB # be sure to convert before passing in size_in_mb = int(volume['size']) * units.Ki try: volutils.copy_volume(self.local_path(volume), self.local_path(volume, vg=dest_vg), size_in_mb, self.configuration.volume_dd_blocksize, execute=self._execute, sparse=self._sparse_copy_volume) except Exception as e: with excutils.save_and_reraise_exception(): LOG.error(_LE("Volume migration failed due to " "exception: %(reason)s."), {'reason': six.text_type(e)}, resource=volume) dest_vg_ref.delete(volume) self._delete_volume(volume) return (True, None)
def _copy_vdisk_data(self, src_vdisk_name, src_vdisk_id, dest_vdisk_name, dest_vdisk_id): """Copy data from src vdisk to dest vdisk. To be able to copy data between vdisks, we must ensure that both vdisks have been mapped to host. If vdisk has not been mapped, it must be mapped firstly. When data copy completed, vdisk should be restored to previous mapped or non-mapped status. """ LOG.debug('enter: _copy_vdisk_data: %(src)s -> %(dest)s.', { 'src': src_vdisk_name, 'dest': dest_vdisk_name }) connector = utils.brick_get_connector_properties() (src_map, src_lun_id) = self._is_vdisk_map(src_vdisk_name, connector) (dest_map, dest_lun_id) = self._is_vdisk_map(dest_vdisk_name, connector) src_map_device = None src_properties = None dest_map_device = None dest_properties = None try: if not src_map: src_lun_id = self._map_vdisk_to_host(src_vdisk_name, connector) if not dest_map: dest_lun_id = self._map_vdisk_to_host(dest_vdisk_name, connector) src_properties = self._get_vdisk_map_properties( connector, src_lun_id, src_vdisk_name, src_vdisk_id, self._get_vdisk_params(None)) src_map_device = self._scan_device(src_properties) dest_properties = self._get_vdisk_map_properties( connector, dest_lun_id, dest_vdisk_name, dest_vdisk_id, self._get_vdisk_params(None)) dest_map_device = self._scan_device(dest_properties) src_vdisk_attr = self._get_vdisk_attributes(src_vdisk_name) # vdisk capacity is bytes, translate into MB size_in_mb = int(src_vdisk_attr['capacity']) / units.Mi volume_utils.copy_volume(src_map_device['path'], dest_map_device['path'], size_in_mb, self.configuration.volume_dd_blocksize) except Exception: with excutils.save_and_reraise_exception(): LOG.error('Failed to copy %(src)s to %(dest)s.', { 'src': src_vdisk_name, 'dest': dest_vdisk_name }) finally: if not dest_map: self._unmap_vdisk_from_host(dest_vdisk_name, connector) self._remove_device(dest_properties, dest_map_device) if not src_map: self._unmap_vdisk_from_host(src_vdisk_name, connector) self._remove_device(src_properties, src_map_device) LOG.debug('leave: _copy_vdisk_data: %(src)s -> %(dest)s.', { 'src': src_vdisk_name, 'dest': dest_vdisk_name })
def create_volume_from_snapshot(self, volume, snapshot): """Create volume from snapshot. - search for snapshot and retention_policy - create a view from snapshot and attach view - create a volume and attach volume - copy data from attached view to attached volume - detach volume and view and finally delete view """ snap_name = self.get_snap_name(snapshot.id) view_name = self.get_view_name(volume.id) vol_name = self.get_volume_name(volume.id) cview = src_attach_info = dest_attach_info = None rpolicy = self.get_policy() properties = utils.brick_get_connector_properties() LOG.debug("Searching for snapshot: %s in K2.", snap_name) snap_rs = self.client.search("snapshots", short_name=snap_name) if hasattr(snap_rs, 'hits') and snap_rs.total != 0: snap = snap_rs.hits[0] LOG.debug("Creating a view: %(view)s from snapshot: %(snap)s", {'view': view_name, 'snap': snap_name}) try: cview = self.client.new("snapshots", short_name=view_name, source=snap, retention_policy=rpolicy, is_exposable=True).save() except Exception as ex: LOG.exception("Creating a view: %(view)s from snapshot: " "%(snap)s failed", {"view": view_name, "snap": snap_name}) raise exception.KaminarioCinderDriverException(reason=ex) else: msg = _("Snapshot: %s search failed in K2.") % snap_name LOG.error(msg) raise exception.KaminarioCinderDriverException(reason=msg) try: conn = self.initialize_connection(cview, properties) src_attach_info = self._connect_device(conn) self.create_volume(volume) conn = self.initialize_connection(volume, properties) dest_attach_info = self._connect_device(conn) vol_utils.copy_volume(src_attach_info['device']['path'], dest_attach_info['device']['path'], snapshot.volume.size * units.Ki, self.configuration.volume_dd_blocksize, sparse=True) self._kaminario_disconnect_volume(src_attach_info, dest_attach_info) self.terminate_connection(volume, properties) self.terminate_connection(cview, properties) cview.delete() except Exception as ex: self._kaminario_disconnect_volume(src_attach_info, dest_attach_info) self.terminate_connection(cview, properties) self.terminate_connection(volume, properties) cview.delete() self.delete_volume(volume) LOG.exception("Copy to volume: %(vol)s from view: %(view)s " "failed", {"vol": vol_name, "view": view_name}) raise exception.KaminarioCinderDriverException(reason=ex)
def fetch_to_volume_format(context, image_service, image_id, dest, volume_format, blocksize, user_id=None, project_id=None, size=None): if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) qemu_img = True image_meta = image_service.show(context, image_id) # NOTE(avishay): I'm not crazy about creating temp files which may be # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because # it seeks. Maybe we can think of something for a future version. with temporary_file() as tmp: # We may be on a system that doesn't have qemu-img installed. That # is ok if we are working with a RAW image. This logic checks to see # if qemu-img is installed. If not we make sure the image is RAW and # throw an exception if not. Otherwise we stop before needing # qemu-img. Systems with qemu-img will always progress through the # whole function. try: # Use the empty tmp file to make sure qemu_img_info works. qemu_img_info(tmp) except processutils.ProcessExecutionError: qemu_img = False if image_meta: if image_meta['disk_format'] != 'raw': raise exception.ImageUnacceptable( reason=_("qemu-img is not installed and image is of " "type %s. Only RAW images can be used if " "qemu-img is not installed.") % image_meta['disk_format'], image_id=image_id) else: raise exception.ImageUnacceptable(reason=_( "qemu-img is not installed and the disk " "format is not specified. Only RAW images " "can be used if qemu-img is not installed."), image_id=image_id) fetch(context, image_service, image_id, tmp, user_id, project_id) if is_xenserver_image(context, image_service, image_id): replace_xenserver_image_with_coalesced_vhd(tmp) if not qemu_img: # qemu-img is not installed but we do have a RAW image. As a # result we only need to copy the image to the destination and then # return. LOG.debug('Copying image from %(tmp)s to volume %(dest)s - ' 'size: %(size)s' % { 'tmp': tmp, 'dest': dest, 'size': image_meta['size'] }) volume_utils.copy_volume(tmp, dest, image_meta['size'], blocksize) return data = qemu_img_info(tmp) virt_size = data.virtual_size / units.Gi # NOTE(xqueralt): If the image virtual size doesn't fit in the # requested volume there is no point on resizing it because it will # generate an unusable image. if size is not None and virt_size > size: params = {'image_size': virt_size, 'volume_size': size} reason = _("Size is %(image_size)dGB and doesn't fit in a " "volume of size %(volume_size)dGB.") % params raise exception.ImageUnacceptable(image_id=image_id, reason=reason) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % { 'fmt': fmt, 'backing_file': backing_file, }) # NOTE(jdg): I'm using qemu-img convert to write # to the volume regardless if it *needs* conversion or not # TODO(avishay): We can speed this up by checking if the image is raw # and if so, writing directly to the device. However, we need to keep # check via 'qemu-img info' that what we copied was in fact a raw # image and not a different format with a backing file, which may be # malicious. LOG.debug("%s was %s, converting to %s " % (image_id, fmt, volume_format)) convert_image(tmp, dest, volume_format, bps_limit=CONF.volume_copy_bps_limit) data = qemu_img_info(dest) if data.file_format != volume_format: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(vol_format)s, but format is " "now %(file_format)s") % { 'vol_format': volume_format, 'file_format': data.file_format })