def migrate_volume(self, ctxt, volume, host, thin=False, mirror_count=0):
"""Optimize the migration if the destination is on the same server.
If the specified host is another back-end on the same server, and
the volume is not attached, we can do the migration locally without
going through iSCSI.
"""
false_ret = (False, None)
if volume['status'] != 'available':
return false_ret
if 'location_info' not in host['capabilities']:
return false_ret
info = host['capabilities']['location_info']
try:
(dest_type, dest_hostname, dest_vg, lvm_type, lvm_mirrors) =\
info.split(':')
lvm_mirrors = int(lvm_mirrors)
except ValueError:
return false_ret
if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname):
return false_ret
if dest_vg != self.vg.vg_name:
vg_list = volutils.get_all_volume_groups()
try:
(vg for vg in vg_list if vg['name'] == dest_vg).next()
except StopIteration:
message = (_("Destination Volume Group %s does not exist") %
dest_vg)
LOG.error(message)
return false_ret
helper = utils.get_root_helper()
dest_vg_ref = lvm.LVM(dest_vg, helper,
lvm_type=lvm_type,
executor=self._execute)
self.remove_export(ctxt, volume)
self._create_volume(volume['name'],
self._sizestr(volume['size']),
lvm_type,
lvm_mirrors,
dest_vg_ref)
volutils.copy_volume(self.local_path(volume),
self.local_path(volume, vg=dest_vg),
volume['size'],
self.configuration.volume_dd_blocksize,
execute=self._execute)
self._delete_volume(volume)
model_update = self._create_export(ctxt, volume, vg=dest_vg)
return (True, model_update)
else:
message = (_("Refusing to migrate volume ID: %(id)s. Please "
"check your configuration because source and "
"destination are the same Volume Group: %(name)s.")
% {'id': volume['id'], 'name': self.vg.vg_name})
LOG.exception(message)
raise exception.VolumeBackendAPIException(data=message)
def create_cloned_volume(self, volume, src_vref):
"""Create a cloned volume from an existing one.
No cloning operation in the current release so simply copy using
DD to a new space. This could be a lengthy operation.
"""
# Connect to source volume
volname = self._get_space_name(src_vref)
self._add_cinder_apphost(volname)
# Make new volume
provider = self.create_volume(volume)
self._add_cinder_apphost(provider['provider_id'])
# And copy original into it...
info = self._get_space_size_redundancy(volname)
volutils.copy_volume(
self.local_path(src_vref),
"/dev/" + provider['provider_id'],
info['sizeBytes'] / units.Mi,
self.configuration.volume_dd_blocksize,
execute=self._execute)
# That's all, folks!
return provider
def test_copy_volume_dd_iflag_and_oflag(self, mock_conf, mock_exec, mock_support):
fake_throttle = throttling.Throttle(["fake_throttle"])
output = volume_utils.copy_volume(
"/dev/zero", "/dev/null", 1024, "3M", sync=True, execute=utils.execute, ionice=None, throttle=fake_throttle
)
self.assertIsNone(output)
mock_exec.assert_called_once_with(
"fake_throttle",
"dd",
"if=/dev/zero",
"of=/dev/null",
"count=%s" % units.Gi,
"bs=3M",
"iflag=count_bytes,direct",
"oflag=direct",
run_as_root=True,
)
mock_exec.reset_mock()
output = volume_utils.copy_volume(
"/dev/zero", "/dev/null", 1024, "3M", sync=False, execute=utils.execute, ionice=None, throttle=fake_throttle
)
self.assertIsNone(output)
mock_exec.assert_called_once_with(
"fake_throttle",
"dd",
"if=/dev/zero",
"of=/dev/null",
"count=%s" % units.Gi,
"bs=3M",
"iflag=count_bytes,direct",
"oflag=direct",
run_as_root=True,
)
def copy_sync_data(self, src_ldev, dest_ldev, size):
src_vol = {'provider_location': six.text_type(src_ldev),
'id': 'src_vol'}
dest_vol = {'provider_location': six.text_type(dest_ldev),
'id': 'dest_vol'}
properties = utils.brick_get_connector_properties()
driver = self.generated_from
src_info = None
dest_info = None
try:
dest_info = driver._attach_volume(self.context, dest_vol,
properties)
src_info = driver._attach_volume(self.context, src_vol,
properties)
volume_utils.copy_volume(src_info['device']['path'],
dest_info['device']['path'], size * 1024,
self.configuration.volume_dd_blocksize)
finally:
if dest_info:
driver._detach_volume(self.context, dest_info,
dest_vol, properties)
if src_info:
driver._detach_volume(self.context, src_info,
src_vol, properties)
self.command.discard_zero_page(dest_ldev)
def test_copy_volume_dd_no_iflag_or_oflag(self, mock_exec, mock_support):
fake_throttle = throttling.Throttle(['fake_throttle'])
output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, '3M',
sync=True, execute=utils.execute,
ionice=None, throttle=fake_throttle)
self.assertIsNone(output)
mock_exec.assert_called_once_with('fake_throttle', 'dd',
'if=/dev/zero',
'of=/dev/null',
'count=%s' % units.Gi,
'bs=3M', 'iflag=count_bytes',
'conv=fdatasync', run_as_root=True)
mock_exec.reset_mock()
output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, '3M',
sync=False, execute=utils.execute,
ionice=None, throttle=fake_throttle)
self.assertIsNone(output)
mock_exec.assert_called_once_with('fake_throttle', 'dd',
'if=/dev/zero',
'of=/dev/null',
'count=%s' % units.Gi,
'bs=3M', 'iflag=count_bytes',
run_as_root=True)
def create_volume_from_snapshot(self, volume, snapshot):
"""Create volume from snapshot.
InfiniBox does not yet support detached clone so use dd to copy data.
This could be a lengthy operation.
- create a clone from snapshot and map it
- create a volume and map it
- copy data from clone to volume
- unmap volume and clone and delete the clone
"""
infinidat_snapshot = self._get_infinidat_snapshot(snapshot)
clone_name = self._make_volume_name(volume) + '-internal'
infinidat_clone = infinidat_snapshot.create_snapshot(name=clone_name)
# we need a cinder-volume-like object to map the clone by name
# (which is derived from the cinder id) but the clone is internal
# so there is no such object. mock one
clone = mock.Mock(id=str(volume.id) + '-internal')
try:
infinidat_volume = self._create_volume(volume)
try:
src_ctx = self._device_connect_context(clone)
dst_ctx = self._device_connect_context(volume)
with src_ctx as src_dev, dst_ctx as dst_dev:
dd_block_size = self.configuration.volume_dd_blocksize
vol_utils.copy_volume(src_dev['device']['path'],
dst_dev['device']['path'],
snapshot.volume.size * units.Ki,
dd_block_size,
sparse=True)
except Exception:
infinidat_volume.delete()
raise
finally:
infinidat_clone.delete()
def create_volume_from_snapshot(self, volume, snapshot):
"""Creates a volume from a snapshot."""
self._create_volume(volume['name'], self._sizestr(volume['size']))
volutils.copy_volume(self.local_path(snapshot),
self.local_path(volume),
snapshot['volume_size'] * 1024,
execute=self._execute)
def migrate_volume(self, ctxt, volume, host):
"""Optimize the migration if the destination is on the same server.
If the specified host is another back-end on the same server, and
the volume is not attached, we can do the migration locally without
going through iSCSI.
"""
false_ret = (False, None)
if 'location_info' not in host['capabilities']:
return false_ret
info = host['capabilities']['location_info']
try:
(dest_type, dest_hostname, dest_vg) = info.split(':')
except ValueError:
return false_ret
if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname):
return false_ret
self.remove_export(ctxt, volume)
self._create_volume(volume['name'],
self._sizestr(volume['size']),
dest_vg)
volutils.copy_volume(self.local_path(volume),
self.local_path(volume, vg=dest_vg),
volume['size'],
execute=self._execute)
self._delete_volume(volume)
model_update = self._create_export(ctxt, volume, vg=dest_vg)
return (True, model_update)
def create_cloned_volume(self, volume, src_vref):
"""Creates a clone of the specified volume."""
mirror_count = 0
if self.configuration.lvm_mirrors:
mirror_count = self.configuration.lvm_mirrors
LOG.info(_LI('Creating clone of volume: %s'), src_vref['id'])
volume_name = src_vref['name']
temp_id = 'tmp-snap-%s' % volume['id']
temp_snapshot = {'volume_name': volume_name,
'size': src_vref['size'],
'volume_size': src_vref['size'],
'name': 'clone-snap-%s' % volume['id'],
'id': temp_id}
self.create_snapshot(temp_snapshot)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
try:
self._create_volume(volume['name'],
self._sizestr(volume['size']),
self.configuration.lvm_type,
mirror_count)
self.vg.activate_lv(temp_snapshot['name'], is_snapshot=True)
volutils.copy_volume(
self.local_path(temp_snapshot),
self.local_path(volume),
src_vref['size'] * units.Ki,
self.configuration.volume_dd_blocksize,
execute=self._execute,
sparse=self.sparse_copy_volume)
finally:
self.delete_snapshot(temp_snapshot)
def create_snapshot(self, snapshot):
"""Create a snapshot volume.
We don't yet support snaps in SW so make a new volume and dd the
source one into it. This could be a lengthy operation.
"""
origvol = {}
origvol['name'] = snapshot['volume_name']
origvol['size'] = snapshot['volume_size']
origvol['id'] = snapshot['volume_id']
origvol['provider_id'] = snapshot.get('volume').get('provider_id')
# Add me to the apphosts so I can see the volume
self._add_cinder_apphost(self._get_space_name(origvol))
# Make snapshot volume
snapvol = {}
snapvol['display_name'] = snapshot['display_name']
snapvol['name'] = snapshot['name']
snapvol['size'] = snapshot['volume_size']
snapvol['id'] = snapshot['id']
provider = self.create_volume(snapvol)
# Create_volume attaches the volume to this host, ready to snapshot.
# Copy it using dd for now, we don't have real snapshots
# We need to copy the entire allocated volume space, Nova will allow
# full access, even beyond requested size (when our volume is larger
# due to our ~1B byte alignment or cluster makeup)
info = self._get_space_size_redundancy(origvol['provider_id'])
volutils.copy_volume(
self.local_path(origvol),
"/dev/" + provider['provider_id'],
info['sizeBytes'] / units.Mi,
self.configuration.volume_dd_blocksize,
execute=self._execute)
return provider
def test_copy_volume_dd_no_iflag_or_oflag(self, mock_exec, mock_support, mock_count):
fake_throttle = throttling.Throttle(["fake_throttle"])
output = volume_utils.copy_volume(
"/dev/zero", "/dev/null", 1024, 1, sync=True, execute=utils.execute, ionice=None, throttle=fake_throttle
)
self.assertIsNone(output)
mock_exec.assert_called_once_with(
"fake_throttle",
"dd",
"if=/dev/zero",
"of=/dev/null",
"count=5678",
"bs=1234",
"conv=fdatasync",
run_as_root=True,
)
mock_exec.reset_mock()
output = volume_utils.copy_volume(
"/dev/zero", "/dev/null", 1024, 1, sync=False, execute=utils.execute, ionice=None, throttle=fake_throttle
)
self.assertIsNone(output)
mock_exec.assert_called_once_with(
"fake_throttle", "dd", "if=/dev/zero", "of=/dev/null", "count=5678", "bs=1234", run_as_root=True
)
def create_volume_from_snapshot(self, volume, snapshot):
"""Creates a volume from a snapshot."""
if self.configuration.lvm_type == 'thin':
self.vg.create_lv_snapshot(volume['name'],
self._escape_snapshot(snapshot['name']),
self.configuration.lvm_type)
if volume['size'] > snapshot['volume_size']:
LOG.debug("Resize the new volume to %s.", volume['size'])
self.extend_volume(volume, volume['size'])
# Some configurations of LVM do not automatically activate
# ThinLVM snapshot LVs.
self.vg.activate_lv(snapshot['name'], is_snapshot=True)
self.vg.activate_lv(volume['name'], is_snapshot=True,
permanent=True)
return
self._create_volume(volume['name'],
self._sizestr(volume['size']),
self.configuration.lvm_type,
self.configuration.lvm_mirrors)
# Some configurations of LVM do not automatically activate
# ThinLVM snapshot LVs.
self.vg.activate_lv(snapshot['name'], is_snapshot=True)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
volutils.copy_volume(self.local_path(snapshot),
self.local_path(volume),
snapshot['volume_size'] * units.Ki,
self.configuration.volume_dd_blocksize,
execute=self._execute,
sparse=self._sparse_copy_volume)
def test_create_cloned_volume(self, _copy_volume):
TEST_SRC = obj_volume.Volume(id=fake.VOLUME_ID,
name_id=fake.VOLUME_NAME_ID,
size=1,
provider_location='/dev/loop1')
TEST_VOLUME = obj_volume.Volume(name_id=fake.VOLUME2_NAME_ID,
size=1,
display_name='vol1')
with mock.patch.object(self.drv, 'find_appropriate_size_device',
return_value='/dev/loop2') as fasd_mocked:
with mock.patch.object(self.drv, '_get_devices_sizes',
return_value={'/dev/loop2': 2}) as \
gds_mocked:
with mock.patch.object(self.drv, 'local_path',
return_value='/dev/loop1') as \
lp_mocked:
with mock.patch.object(self.drv,
'_update_provider_location') as \
upl_mocked:
volutils.copy_volume('/dev/loop1', fasd_mocked, 2,
mock.sentinel,
execute=self.drv._execute)
self.drv.create_cloned_volume(TEST_VOLUME, TEST_SRC)
fasd_mocked.assert_called_once_with(TEST_SRC.size)
lp_mocked.assert_called_once_with(TEST_SRC)
gds_mocked.assert_called_once_with(['/dev/loop2'])
upl_mocked.assert_called_once_with(
TEST_VOLUME, '/dev/loop2')
def test_create_volume_from_snapshot(self, _copy_volume):
TEST_SNAP = obj_snap.Snapshot(volume_id=fake.VOLUME_ID,
volume_size=1024,
provider_location='/dev/loop1')
TEST_VOLUME = obj_volume.Volume(id=fake.VOLUME_ID,
name_id=fake.VOLUME_NAME_ID,
size=1,
display_name='vol1',
provider_location='/dev/loop2')
with mock.patch.object(self.drv, 'find_appropriate_size_device',
return_value='/dev/loop2') as fasd_mocked:
with mock.patch.object(self.drv, '_get_devices_sizes',
return_value={'/dev/loop2': 1024}) as \
gds_mocked:
with mock.patch.object(self.drv,
'_update_provider_location') as \
upl_mocked:
volutils.copy_volume('/dev/loop1', fasd_mocked, 1024,
mock.sentinel,
execute=self.drv._execute)
self.drv.create_volume_from_snapshot(
TEST_VOLUME, TEST_SNAP)
fasd_mocked.assert_called_once_with(
TEST_SNAP.volume_size)
gds_mocked.assert_called_once_with(['/dev/loop2'])
upl_mocked.assert_called_once_with(
TEST_VOLUME, '/dev/loop2')
def delete_volume(self, volume):
"""
Removes the data from volume and returns the volume to pool.
:param volume: OpenStack Volume Object.
"""
if self.configuration.sl_pool_volume_clear not in ("zero", "shred", "none"):
raise exception.InvalidConfigurationValue(
option="volume_clear", value=self.configuration.sl_pool_volume_clear
)
sl_vol = self.meta_mgr.deserialize(volume["id"])
connection = self.vol_mgr.get_iscsi_properties(sl_vol)
attach_info = self._attch(connection)
size_in_mb = 1024 * volume["size"]
try:
if self.configuration.sl_pool_volume_clear == "zero":
LOG.info("zeroing out volume")
volume_utils.copy_volume("/dev/zero", attach_info["device"]["path"], size_in_mb)
elif self.configuration.sl_pool_volume_clear == "shred":
LOG.info("Shredding volume")
utils.execute("shred", "-n3", "-s%dMiB" % size_in_mb, attach_info["device"]["path"], run_as_root=True)
except proc_utils.ProcessExecutionError as ex:
LOG.error(_("Error while swiping out data. %s" % ex))
raise
finally:
self._detach_volume(attach_info)
self.meta_mgr.delete_all(volume["id"])
def create_cloned_volume(self, volume, src_vref):
"""Creates a clone of the specified volume."""
mirror_count = 0
if self.configuration.lvm_mirrors:
mirror_count = self.configuration.lvm_mirrors
LOG.info(_('Creating clone of volume: %s') % src_vref['id'])
volume_name = src_vref['name']
temp_id = 'tmp-snap-%s' % volume['id']
temp_snapshot = {'volume_name': volume_name,
'size': src_vref['size'],
'volume_size': src_vref['size'],
'name': 'clone-snap-%s' % volume['id'],
'id': temp_id}
self.create_snapshot(temp_snapshot)
self._create_volume(volume['name'],
self._sizestr(volume['size']),
self.configuration.lvm_type,
mirror_count)
self.vg.activate_lv(temp_snapshot['name'], is_snapshot=True)
try:
volutils.copy_volume(
self.local_path(temp_snapshot),
self.local_path(volume),
src_vref['size'] * 1024,
self.configuration.volume_dd_blocksize,
execute=self._execute)
finally:
self.delete_snapshot(temp_snapshot)
def create_cloned_volume(self, volume, src_vref):
"""Create a clone from source volume.
InfiniBox does not yet support detached clone so use dd to copy data.
This could be a lengthy operation.
* map source volume
* create and map new volume
* copy data from source to new volume
* unmap both volumes
"""
self._asssert_volume_not_mapped(src_vref)
infinidat_volume = self._create_volume(volume)
try:
src_ctx = self._device_connect_context(src_vref)
dst_ctx = self._device_connect_context(volume)
with src_ctx as src_dev, dst_ctx as dst_dev:
dd_block_size = self.configuration.volume_dd_blocksize
vol_utils.copy_volume(src_dev['device']['path'],
dst_dev['device']['path'],
src_vref.size * units.Ki,
dd_block_size,
sparse=True)
except Exception:
infinidat_volume.delete()
raise
def create_volume_from_snapshot(self, volume, snapshot):
"""Create volume from snapshot.
- search for snapshot and retention_policy
- create a view from snapshot and attach view
- create a volume and attach volume
- copy data from attached view to attached volume
- detach volume and view and finally delete view
"""
snap_name = self.get_snap_name(snapshot.id)
view_name = self.get_view_name(volume.id)
vol_name = self.get_volume_name(volume.id)
cview = src_attach_info = dest_attach_info = None
rpolicy = self.get_policy()
properties = utils.brick_get_connector_properties()
LOG.debug("Searching for snapshot: %s in K2.", snap_name)
snap_rs = self.client.search("snapshots", short_name=snap_name)
if hasattr(snap_rs, 'hits') and snap_rs.total != 0:
snap = snap_rs.hits[0]
LOG.debug("Creating a view: %(view)s from snapshot: %(snap)s",
{'view': view_name, 'snap': snap_name})
try:
cview = self.client.new("snapshots",
short_name=view_name,
source=snap, retention_policy=rpolicy,
is_exposable=True).save()
except Exception as ex:
LOG.exception(_LE("Creating a view: %(view)s from snapshot: "
"%(snap)s failed"), {"view": view_name,
"snap": snap_name})
raise exception.KaminarioCinderDriverException(
reason=six.text_type(ex.message))
else:
msg = _("Snapshot: %s search failed in K2.") % snap_name
LOG.error(msg)
raise exception.KaminarioCinderDriverException(reason=msg)
try:
conn = self.initialize_connection(cview, properties)
src_attach_info = self._connect_device(conn)
self.create_volume(volume)
conn = self.initialize_connection(volume, properties)
dest_attach_info = self._connect_device(conn)
vol_utils.copy_volume(src_attach_info['device']['path'],
dest_attach_info['device']['path'],
snapshot.volume.size * units.Ki,
self.configuration.volume_dd_blocksize,
sparse=True)
self.terminate_connection(volume, properties)
self.terminate_connection(cview, properties)
except Exception as ex:
self.terminate_connection(cview, properties)
self.terminate_connection(volume, properties)
cview.delete()
self.delete_volume(volume)
LOG.exception(_LE("Copy to volume: %(vol)s from view: %(view)s "
"failed"), {"vol": vol_name, "view": view_name})
raise exception.KaminarioCinderDriverException(
reason=six.text_type(ex.message))
def create_cloned_volume(self, volume, src_vref):
"""Creates a clone of the specified volume."""
mirror_count = 0
if self.configuration.lvm_mirrors:
mirror_count = self.configuration.lvm_mirrors
LOG.info(_("Creating clone of volume: %s") % src_vref["id"])
volume_name = src_vref["name"]
temp_id = "tmp-snap-%s" % volume["id"]
temp_snapshot = {
"volume_name": volume_name,
"size": src_vref["size"],
"volume_size": src_vref["size"],
"name": "clone-snap-%s" % volume["id"],
"id": temp_id,
}
self.create_snapshot(temp_snapshot)
self._create_volume(volume["name"], self._sizestr(volume["size"]), self.configuration.lvm_type, mirror_count)
self.vg.activate_lv(temp_snapshot["name"], is_snapshot=True)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
try:
volutils.copy_volume(
self.local_path(temp_snapshot),
self.local_path(volume),
src_vref["size"] * units.KiB,
self.configuration.volume_dd_blocksize,
execute=self._execute,
)
finally:
self.delete_snapshot(temp_snapshot)
def test_clear_volume_zero(self):
CONF.volume_clear = 'zero'
CONF.volume_clear_size = 1
self.mox.StubOutWithMock(volume_utils, 'copy_volume')
volume_utils.copy_volume("/dev/zero", "volume_path", 1,
CONF.volume_dd_blocksize, sync=True,
execute=utils.execute)
self.mox.ReplayAll()
volume_utils.clear_volume(1024, "volume_path")
def copy_volume_data(self, context, src_vol, dest_vol, remote=None):
"""Copy data from src_vol to dest_vol."""
LOG.debug(_('copy_data_between_volumes %(src)s -> %(dest)s.')
% {'src': src_vol['name'], 'dest': dest_vol['name']})
properties = utils.brick_get_connector_properties()
dest_remote = True if remote in ['dest', 'both'] else False
dest_orig_status = dest_vol['status']
try:
dest_attach_info = self._attach_volume(context,
dest_vol,
properties,
remote=dest_remote)
except Exception:
with excutils.save_and_reraise_exception():
msg = _("Failed to attach volume %(vol)s")
LOG.error(msg % {'vol': dest_vol['id']})
self.db.volume_update(context, dest_vol['id'],
{'status': dest_orig_status})
src_remote = True if remote in ['src', 'both'] else False
src_orig_status = src_vol['status']
try:
src_attach_info = self._attach_volume(context,
src_vol,
properties,
remote=src_remote)
except Exception:
with excutils.save_and_reraise_exception():
msg = _("Failed to attach volume %(vol)s")
LOG.error(msg % {'vol': src_vol['id']})
self.db.volume_update(context, src_vol['id'],
{'status': src_orig_status})
self._copy_volume_data_cleanup(context, dest_vol, properties,
dest_attach_info, dest_remote,
force=True)
try:
size_in_mb = int(src_vol['size']) * 1024 # vol size is in GB
volume_utils.copy_volume(
src_attach_info['device']['path'],
dest_attach_info['device']['path'],
size_in_mb,
self.configuration.volume_dd_blocksize)
copy_error = False
except Exception:
with excutils.save_and_reraise_exception():
msg = _("Failed to copy volume %(src)s to %(dest)d")
LOG.error(msg % {'src': src_vol['id'], 'dest': dest_vol['id']})
copy_error = True
finally:
self._copy_volume_data_cleanup(context, dest_vol, properties,
dest_attach_info, dest_remote,
force=copy_error)
self._copy_volume_data_cleanup(context, src_vol, properties,
src_attach_info, src_remote,
force=copy_error)
def create_cloned_volume(self, volume, src_vref):
LOG.info(_('Creating clone of volume: %s') % src_vref['id'])
device = self.find_appropriate_size_device(src_vref['size'])
volutils.copy_volume(self.local_path(src_vref), device,
self._get_device_size(device) * 2048,
execute=self._execute)
return {
'provider_location': self._iscsi_location(None, None, None, None,
device),
}
def test_clear_volume_zero(self):
CONF.volume_clear = "zero"
CONF.volume_clear_size = 1
CONF.volume_clear_ionice = None
self.mox.StubOutWithMock(volume_utils, "copy_volume")
volume_utils.copy_volume(
"/dev/zero", "volume_path", 1, CONF.volume_dd_blocksize, sync=True, ionice=None, execute=utils.execute
)
self.mox.ReplayAll()
volume_utils.clear_volume(1024, "volume_path")
def create_volume_from_snapshot(self, volume, snapshot):
LOG.info(_LI('Creating volume %s from snapshot.'), volume.id)
device = self.find_appropriate_size_device(snapshot.volume_size)
dev_size = self._get_devices_sizes([device])
volutils.copy_volume(
self.local_path(snapshot), device,
dev_size[device],
self.configuration.volume_dd_blocksize,
execute=self._execute)
self._update_provider_location(volume, device)
def create_cloned_volume(self, volume, src_vref):
LOG.info(_LI('Creating clone of volume: %s.'), src_vref.id)
device = self.find_appropriate_size_device(src_vref.size)
dev_size = self._get_devices_sizes([device])
volutils.copy_volume(
self.local_path(src_vref), device,
dev_size[device],
self.configuration.volume_dd_blocksize,
execute=self._execute)
self._update_provider_location(volume, device)
def migrate_volume(self, ctxt, volume, host, thin=False, mirror_count=0):
"""Optimize the migration if the destination is on the same server.
If the specified host is another back-end on the same server, and
the volume is not attached, we can do the migration locally without
going through iSCSI.
"""
false_ret = (False, None)
if volume['status'] != 'available':
return false_ret
if 'location_info' not in host['capabilities']:
return false_ret
info = host['capabilities']['location_info']
try:
(dest_type, dest_hostname, dest_vg, lvm_type, lvm_mirrors) =\
info.split(':')
lvm_mirrors = int(lvm_mirrors)
except ValueError:
return false_ret
if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname):
return false_ret
if dest_vg != self.vg.vg_name:
vg_list = volutils.get_all_volume_groups()
try:
(vg for vg in vg_list if vg['name'] == dest_vg).next()
except StopIteration:
message = (_("Destination Volume Group %s does not exist") %
dest_vg)
LOG.error(message)
return false_ret
helper = utils.get_root_helper()
dest_vg_ref = lvm.LVM(dest_vg, helper,
lvm_type=lvm_type,
executor=self._execute)
self.remove_export(ctxt, volume)
self._create_volume(volume['name'],
self._sizestr(volume['size']),
lvm_type,
lvm_mirrors,
dest_vg_ref)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
size_in_mb = volume['size'] * units.Ki
volutils.copy_volume(self.local_path(volume),
self.local_path(volume, vg=dest_vg),
size_in_mb,
self.configuration.volume_dd_blocksize,
execute=self._execute)
self._delete_volume(volume)
model_update = self._create_export(ctxt, volume, vg=dest_vg)
return (True, model_update)
def create_cloned_volume(self, volume, src_vref):
LOG.info(_LI('Creating clone of volume: %s'), src_vref['id'])
device = self.find_appropriate_size_device(src_vref['size'])
volutils.copy_volume(
self.local_path(src_vref), device,
self._get_device_size(device) * 2048,
self.configuration.volume_dd_blocksize,
execute=self._execute)
return {
'provider_location': device,
}
def create_cloned_volume(self, volume, src_vref):
LOG.info(_("Creating clone of volume: %s") % src_vref["id"])
device = self.find_appropriate_size_device(src_vref["size"])
volutils.copy_volume(
self.local_path(src_vref),
device,
self._get_device_size(device) * 2048,
self.configuration.volume_dd_blocksize,
execute=self._execute,
)
return {"provider_location": device}
def _copy_vdisk_data(self, src_vdisk_name, src_vdisk_id, dest_vdisk_name, dest_vdisk_id):
"""Copy data from src vdisk to dest vdisk.
To be able to copy data between vdisks, we must ensure that both
vdisks have been mapped to host. If vdisk has not been mapped,
it must be mapped firstly. When data copy completed, vdisk
should be restored to previous mapped or non-mapped status.
"""
LOG.debug("enter: _copy_vdisk_data: %(src)s -> %(dest)s.", {"src": src_vdisk_name, "dest": dest_vdisk_name})
connector = utils.brick_get_connector_properties()
(src_map, src_lun_id) = self._is_vdisk_map(src_vdisk_name, connector)
(dest_map, dest_lun_id) = self._is_vdisk_map(dest_vdisk_name, connector)
src_map_device = None
src_properties = None
dest_map_device = None
dest_properties = None
try:
if not src_map:
src_lun_id = self._map_vdisk_to_host(src_vdisk_name, connector)
if not dest_map:
dest_lun_id = self._map_vdisk_to_host(dest_vdisk_name, connector)
src_properties = self._get_vdisk_map_properties(
connector, src_lun_id, src_vdisk_name, src_vdisk_id, self._get_vdisk_params(None)
)
src_map_device = self._scan_device(src_properties)
dest_properties = self._get_vdisk_map_properties(
connector, dest_lun_id, dest_vdisk_name, dest_vdisk_id, self._get_vdisk_params(None)
)
dest_map_device = self._scan_device(dest_properties)
src_vdisk_attr = self._get_vdisk_attributes(src_vdisk_name)
# vdisk capacity is bytes, translate into MB
size_in_mb = int(src_vdisk_attr["capacity"]) / units.Mi
volume_utils.copy_volume(
src_map_device["path"], dest_map_device["path"], size_in_mb, self.configuration.volume_dd_blocksize
)
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Failed to copy %(src)s to %(dest)s."), {"src": src_vdisk_name, "dest": dest_vdisk_name})
finally:
if not dest_map:
self._unmap_vdisk_from_host(dest_vdisk_name, connector)
self._remove_device(dest_properties, dest_map_device)
if not src_map:
self._unmap_vdisk_from_host(src_vdisk_name, connector)
self._remove_device(src_properties, src_map_device)
LOG.debug("leave: _copy_vdisk_data: %(src)s -> %(dest)s.", {"src": src_vdisk_name, "dest": dest_vdisk_name})
def test_copy_volume_dd_iflag_and_oflag(self):
def fake_utils_execute(*cmd, **kwargs):
if 'if=/dev/zero' in cmd and 'iflag=direct' in cmd:
raise processutils.ProcessExecutionError()
if 'of=/dev/null' in cmd and 'oflag=direct' in cmd:
raise processutils.ProcessExecutionError()
if 'iflag=direct' in cmd and 'oflag=direct' in cmd:
raise exception.InvalidInput(message='iflag/oflag error')
volume_utils.copy_volume('/dev/zero', '/dev/null', 1024,
CONF.volume_dd_blocksize, sync=True,
ionice=None, execute=fake_utils_execute)
def _dd_copy(self, vol_params, src_snap, src_lun=None):
"""Creates a volume via copying a Unity snapshot.
It attaches the `volume` and `snap`, then use `dd` to copy the
data from the Unity snapshot to the `volume`.
"""
dest_lun = self.client.create_lun(
name=vol_params.name, size=vol_params.size, pool=vol_params.pool,
description=vol_params.description,
io_limit_policy=vol_params.io_limit_policy)
src_id = src_snap.get_id()
try:
conn_props = cinder_utils.brick_get_connector_properties()
with self._connect_resource(dest_lun, conn_props,
vol_params.volume_id) as dest_info, \
self._connect_resource(src_snap, conn_props,
src_id) as src_info:
if src_lun is None:
# If size is not specified, need to get the size from LUN
# of snapshot.
lun = self.client.get_lun(
lun_id=src_snap.storage_resource.get_id())
size_in_m = utils.byte_to_mib(lun.size_total)
else:
size_in_m = utils.byte_to_mib(src_lun.size_total)
vol_utils.copy_volume(
src_info['device']['path'],
dest_info['device']['path'],
size_in_m,
self.driver.configuration.volume_dd_blocksize,
sparse=True)
except Exception:
with excutils.save_and_reraise_exception():
utils.ignore_exception(self.client.delete_lun,
dest_lun.get_id())
LOG.error('Failed to create cloned volume: %(vol_id)s, '
'from source unity snapshot: %(snap_name)s.',
{'vol_id': vol_params.volume_id,
'snap_name': src_snap.name})
return dest_lun
def test_copy_volume_dd_iflag_and_oflag(self, mock_conf, mock_exec,
mock_support, mock_count):
fake_throttle = throttling.Throttle(['fake_throttle'])
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
1,
sync=True,
execute=utils.execute,
ionice=None,
throttle=fake_throttle)
self.assertIsNone(output)
mock_exec.assert_called_once_with('fake_throttle',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=5678',
'bs=1234',
'iflag=direct',
'oflag=direct',
run_as_root=True)
mock_exec.reset_mock()
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
1,
sync=False,
execute=utils.execute,
ionice=None,
throttle=fake_throttle)
self.assertIsNone(output)
mock_exec.assert_called_once_with('fake_throttle',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=5678',
'bs=1234',
'iflag=direct',
'oflag=direct',
run_as_root=True)
def test_copy_volume_dd_with_sparse_iflag_and_oflag(self, mock_exec,
mock_support,
mock_count):
output = volume_utils.copy_volume('/dev/zero', '/dev/null', 1024, 1,
sync=True, execute=utils.execute,
sparse=True)
self.assertIsNone(output)
mock_exec.assert_called_once_with('dd', 'if=/dev/zero', 'of=/dev/null',
'count=5678', 'bs=1234',
'iflag=direct', 'oflag=direct',
'conv=sparse', run_as_root=True)
def create_volume_from_snapshot(self, volume, snapshot):
"""Creates a volume from a snapshot."""
self._create_volume(volume['name'],
self._sizestr(volume['size']),
self.configuration.lvm_type,
self.configuration.lvm_mirrors)
# Terrifying patch to ignore snapshoting when migrating from ceph volume_type_id
if volume['volume_type_id'] != 'b07b4b6d-ac74-4d0b-9e68-cc1a637798b0':
# Some configurations of LVM do not automatically activate
# ThinLVM snapshot LVs.
self.vg.activate_lv(snapshot['name'], is_snapshot=True)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
volutils.copy_volume(self.local_path(snapshot),
self.local_path(volume),
snapshot['volume_size'] * units.Ki,
self.configuration.volume_dd_blocksize,
execute=self._execute,
sparse=self.sparse_copy_volume)
def _copy_with_dd(self, src_ldev, dest_ldev, size):
"""Copy the content of a volume by dd command."""
src_info = None
dest_info = None
properties = cinder_utils.brick_get_connector_properties(
multipath=self.conf.use_multipath_for_image_xfer,
enforce_multipath=self.conf.enforce_multipath_for_image_xfer)
try:
dest_info = self._attach_ldev(dest_ldev, properties)
src_info = self._attach_ldev(src_ldev, properties)
volume_utils.copy_volume(src_info['device']['path'],
dest_info['device']['path'],
size * units.Ki,
self.conf.volume_dd_blocksize)
finally:
if src_info:
self._detach_ldev(src_info, src_ldev, properties)
if dest_info:
self._detach_ldev(dest_info, dest_ldev, properties)
self.discard_zero_page({'provider_location': six.text_type(dest_ldev)})
def _create_volume_from_snap(self, volume, snap, size_in_m=None):
"""Creates a volume from a Unity snapshot.
It attaches the `volume` and `snap`, then use `dd` to copy the
data from the Unity snapshot to the `volume`.
"""
model_update = self.create_volume(volume)
# Update `provider_location` and `provider_id` of `volume` explicitly.
volume.update(model_update)
src_id = snap.get_id()
dest_lun = self.client.get_lun(lun_id=self.get_lun_id(volume))
try:
conn_props = cinder_utils.brick_get_connector_properties()
with self._connect_resource(dest_lun, conn_props,
volume.id) as dest_info, \
self._connect_resource(snap, conn_props,
src_id) as src_info:
if size_in_m is None:
# If size is not specified, need to get the size from LUN
# of snapshot.
lun = self.client.get_lun(
lun_id=snap.storage_resource.get_id())
size_in_m = utils.byte_to_mib(lun.size_total)
vol_utils.copy_volume(
src_info['device']['path'],
dest_info['device']['path'],
size_in_m,
self.driver.configuration.volume_dd_blocksize,
sparse=True)
except Exception:
with excutils.save_and_reraise_exception():
utils.ignore_exception(self.delete_volume, volume)
LOG.error(
'Failed to create cloned volume: %(vol_id)s, '
'from source unity snapshot: %(snap_name)s.', {
'vol_id': volume.id,
'snap_name': snap.name
})
return model_update
def clear_volume(self, volume, is_snapshot=False):
"""unprovision old volumes to prevent data leaking between users."""
# NOTE(jdg): Don't write the blocks of thin provisioned
# volumes
if self.configuration.volume_clear == 'none' or \
self.configuration.lvm_type == 'thin':
return
if is_snapshot:
# if the volume to be cleared is a snapshot of another volume
# we need to clear out the volume using the -cow instead of the
# directly volume path. We need to skip this if we are using
# thin provisioned LVs.
# bug# lp1191812
dev_path = self.local_path(volume) + "-cow"
else:
dev_path = self.local_path(volume)
if not os.path.exists(dev_path):
msg = (_('Volume device file path %s does not exist.') % dev_path)
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
size_in_g = volume.get('size', volume.get('volume_size', None))
if size_in_g is None:
msg = (_("Size for volume: %s not found, "
"cannot secure delete.") % volume['id'])
LOG.error(msg)
raise exception.InvalidParameterValue(msg)
size_in_m = self.configuration.volume_clear_size
LOG.info(_("Performing secure delete on volume: %s") % volume['id'])
if self.configuration.volume_clear == 'zero':
if size_in_m == 0:
return volutils.copy_volume('/dev/zero',
dev_path,
size_in_g * 1024,
sync=True,
execute=self._execute)
else:
clear_cmd = ['shred', '-n0', '-z', '-s%dMiB' % size_in_m]
elif self.configuration.volume_clear == 'shred':
clear_cmd = ['shred', '-n3']
if size_in_m:
clear_cmd.append('-s%dMiB' % size_in_m)
else:
raise exception.InvalidConfigurationValue(
option='volume_clear', value=self.configuration.volume_clear)
clear_cmd.append(dev_path)
self._execute(*clear_cmd, run_as_root=True)
def create_cloned_volume(self, volume, src_vref):
"""Creates a clone of the specified volume."""
LOG.info(_('Creating clone of volume: %s') % src_vref['id'])
volume_name = src_vref['name']
temp_id = 'tmp-snap-%s' % volume['id']
temp_snapshot = {
'volume_name': volume_name,
'size': src_vref['size'],
'volume_size': src_vref['size'],
'name': 'clone-snap-%s' % volume['id'],
'id': temp_id
}
self.create_snapshot(temp_snapshot)
self._create_volume(volume['name'], self._sizestr(volume['size']))
try:
volutils.copy_volume(self.local_path(temp_snapshot),
self.local_path(volume),
src_vref['size'] * 1024,
execute=self._execute)
finally:
self.delete_snapshot(temp_snapshot)
def test_copy_volume_dd_iflag_and_oflag(self, mock_conf, mock_exec,
mock_support, mock_count, mock_cg):
mock_conf.volume_copy_bps_limit = 10
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
1,
sync=True,
execute=utils.execute,
ionice=None)
self.assertIsNone(output)
mock_exec.assert_called_once_with('cg_cmd',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=5678',
'bs=1234',
'iflag=direct',
'oflag=direct',
run_as_root=True)
mock_exec.reset_mock()
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
1,
sync=False,
execute=utils.execute,
ionice=None)
self.assertIsNone(output)
mock_exec.assert_called_once_with('cg_cmd',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=5678',
'bs=1234',
'iflag=direct',
'oflag=direct',
run_as_root=True)
def test_copy_volume_dd_no_iflag_or_oflag(self, mock_exec, mock_support):
fake_throttle = throttling.Throttle(['fake_throttle'])
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
'3M',
sync=True,
execute=utils.execute,
ionice=None,
throttle=fake_throttle)
self.assertIsNone(output)
mock_exec.assert_called_once_with('fake_throttle',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=%s' % units.Gi,
'bs=3M',
'iflag=count_bytes',
'conv=fdatasync',
run_as_root=True)
mock_exec.reset_mock()
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
'3M',
sync=False,
execute=utils.execute,
ionice=None,
throttle=fake_throttle)
self.assertIsNone(output)
mock_exec.assert_called_once_with('fake_throttle',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=%s' % units.Gi,
'bs=3M',
'iflag=count_bytes',
run_as_root=True)
def create_cloned_volume(self, volume, src_vref):
"""Create a clone from source volume.
- attach source volume
- create and attach new volume
- copy data from attached source volume to attached new volume
- detach both volumes
"""
clone_name = self.get_volume_name(volume.id)
src_name = self.get_volume_name(src_vref.id)
src_vol = self.client.search("volumes", name=src_name)
src_map = self.client.search("mappings", volume=src_vol)
if src_map.total != 0:
msg = _("K2 driver does not support clone of a attached volume. "
"To get this done, create a snapshot from the attached "
"volume and then create a volume from the snapshot.")
LOG.error(msg)
raise exception.KaminarioCinderDriverException(reason=msg)
try:
properties = utils.brick_get_connector_properties()
conn = self.initialize_connection(src_vref, properties)
src_attach_info = self._connect_device(conn)
self.create_volume(volume)
conn = self.initialize_connection(volume, properties)
dest_attach_info = self._connect_device(conn)
vol_utils.copy_volume(src_attach_info['device']['path'],
dest_attach_info['device']['path'],
src_vref.size * units.Ki,
self.configuration.volume_dd_blocksize,
sparse=True)
self.terminate_connection(volume, properties)
self.terminate_connection(src_vref, properties)
except Exception as ex:
self.terminate_connection(src_vref, properties)
self.terminate_connection(volume, properties)
self.delete_volume(volume)
LOG.exception(_LE("Create a clone: %s failed."), clone_name)
raise exception.KaminarioCinderDriverException(
reason=six.text_type(ex.message))
def test_copy_volume_dd_iflag_and_oflag(self):
def fake_utils_execute(*cmd, **kwargs):
if 'if=/dev/zero' in cmd and 'iflag=direct' in cmd:
raise processutils.ProcessExecutionError()
if 'of=/dev/null' in cmd and 'oflag=direct' in cmd:
raise processutils.ProcessExecutionError()
if 'iflag=direct' in cmd and 'oflag=direct' in cmd:
raise exception.InvalidInput(message='iflag/oflag error')
def fake_check_odirect(src, dest, flags='blah'):
return False
self.stubs.Set(volume_utils, 'check_for_odirect_support',
fake_check_odirect)
volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
CONF.volume_dd_blocksize,
sync=True,
ionice=None,
execute=fake_utils_execute)
def test_create_cloned_volume(self, _copy_volume):
TEST_SRC = {'id': '1',
'size': 1,
'provider_location': '1 2 3 /dev/loop1'}
TEST_VOLUME = {}
with mock.patch.object(self.drv, 'find_appropriate_size_device',
return_value='/dev/loop2') as fasd_mocked:
with mock.patch.object(self.drv, '_get_device_size',
return_value=1) as gds_mocked:
with mock.patch.object(self.drv, 'local_path',
return_value='/dev/loop1') as \
lp_mocked:
volutils.copy_volume('/dev/loop1', fasd_mocked, 2048,
mock.sentinel,
execute=self.drv._execute)
self.assertEqual(self.drv.create_cloned_volume(
TEST_VOLUME, TEST_SRC),
{'provider_location': '/dev/loop2'})
fasd_mocked.assert_called_once_with(TEST_SRC['size'])
lp_mocked.assert_called_once_with(TEST_SRC)
gds_mocked.assert_called_once_with('/dev/loop2')
def _create_and_copy_volume(self, dstvol, srcvol):
"""Creates a volume from a volume or a snapshot."""
updates = self._create_file(dstvol)
# We need devices attached for IO operations.
with temp_lvm_device(self, srcvol) as vg, \
temp_raw_device(self, dstvol):
self._setup_lvm(dstvol)
# Some configurations of LVM do not automatically activate
# ThinLVM snapshot LVs.
with patched(lvm.LVM, 'activate_lv', self._activate_lv):
vg.activate_lv(srcvol['name'], True)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
volutils.copy_volume(self._mapper_path(srcvol),
self._mapper_path(dstvol),
srcvol['volume_size'] * units.Ki,
self.configuration.volume_dd_blocksize,
execute=self._execute)
return updates
def test_create_cloned_volume(self):
TEST_SRC = {
'id': '1',
'size': 1,
'provider_location': '1 2 3 /dev/loop1'
}
TEST_VOLUME = {}
self.mox.StubOutWithMock(self.drv, 'find_appropriate_size_device')
dev = self.drv.find_appropriate_size_device(TEST_SRC['size']).\
AndReturn('/dev/loop2')
self.mox.StubOutWithMock(volutils, 'copy_volume')
self.mox.StubOutWithMock(self.drv, 'local_path')
self.mox.StubOutWithMock(self.drv, '_get_device_size')
self.drv.local_path(TEST_SRC).AndReturn('/dev/loop1')
self.drv._get_device_size('/dev/loop2').AndReturn(1)
volutils.copy_volume('/dev/loop1',
dev,
2048,
mox.IgnoreArg(),
execute=self.drv._execute)
self.mox.ReplayAll()
self.assertEqual(self.drv.create_cloned_volume(TEST_VOLUME, TEST_SRC),
{'provider_location': '/dev/loop2'})
def create_cloned_volume(self, volume, src_vref):
"""Creates a clone of the specified volume."""
mirror_count = 0
if self.configuration.lvm_mirrors:
mirror_count = self.configuration.lvm_mirrors
LOG.info(_LI('Creating clone of volume: %s'), src_vref['id'])
volume_name = src_vref['name']
temp_id = 'tmp-snap-%s' % volume['id']
temp_snapshot = {'volume_name': volume_name,
'size': src_vref['size'],
'volume_size': src_vref['size'],
'name': 'clone-snap-%s' % volume['id'],
'id': temp_id}
self.create_snapshot(temp_snapshot)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
try:
self._create_volume(volume['name'],
self._sizestr(volume['size']),
self.configuration.lvm_type,
mirror_count)
self.vg.activate_lv(temp_snapshot['name'], is_snapshot=True)
sparse = True if self.configuration.lvm_type == 'thin' else False
volutils.copy_volume(
self.local_path(temp_snapshot),
self.local_path(volume),
src_vref['size'] * units.Ki,
self.configuration.volume_dd_blocksize,
execute=self._execute,
sparse=sparse)
finally:
self.delete_snapshot(temp_snapshot)
def create_volume_from_snapshot(self, volume, snapshot):
"""Create volume from snapshot.
InfiniBox does not yet support detached clone so use dd to copy data.
This could be a lengthy operation.
- create a clone from snapshot and map it
- create a volume and map it
- copy data from clone to volume
- unmap volume and clone and delete the clone
"""
snapshot_id = self._get_infinidat_snapshot_id(snapshot)
clone_name = self._make_volume_name(volume) + '-internal'
infinidat_clone = self._post(
'volumes', dict(parent_id=snapshot_id, name=clone_name))
# we need a cinder-volume-like object to map the clone by name
# (which is derived from the cinder id) but the clone is internal
# so there is no such object. mock one
clone = mock.Mock(id=str(volume.id) + '-internal')
try:
infinidat_volume = self._create_volume(volume)
try:
src_ctx = self._device_connect_context(clone)
dst_ctx = self._device_connect_context(volume)
with src_ctx as src_dev, dst_ctx as dst_dev:
dd_block_size = self.configuration.volume_dd_blocksize
vol_utils.copy_volume(src_dev['device']['path'],
dst_dev['device']['path'],
snapshot.volume.size * units.Ki,
dd_block_size,
sparse=True)
except Exception:
self._delete(DELETE_URI % infinidat_volume['id'])
raise
finally:
self._delete(DELETE_URI % infinidat_clone['id'])
def create_cloned_volume(self, volume, src_vref):
"""Create a cloned volume from an existing one.
No cloning operation in the current release so simply copy using
DD to a new space. This could be a lengthy operation.
"""
# Connect to source volume
volname = self._get_space_name(src_vref)
self._add_cinder_apphost(volname)
# Make new volume
provider = self.create_volume(volume)
self._add_cinder_apphost(provider['provider_id'])
# And copy original into it...
info = self._get_space_size_redundancy(volname)
volutils.copy_volume(self.local_path(src_vref),
"/dev/" + provider['provider_id'],
info['sizeBytes'] // units.Mi,
self.configuration.volume_dd_blocksize,
execute=self._execute)
# That's all, folks!
return provider
def test_copy_volume_dd_no_throttle(self, mock_exec, mock_support,
mock_count):
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
1,
sync=True,
execute=utils.execute,
ionice=None)
self.assertIsNone(output)
mock_exec.assert_called_once_with('dd',
'if=/dev/zero',
'of=/dev/null',
'count=5678',
'bs=1234',
'conv=fdatasync',
run_as_root=True)
def test_copy_volume_dd_with_sparse(self, mock_exec, mock_support):
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
'3M',
sync=True,
execute=utils.execute,
sparse=True)
self.assertIsNone(output)
mock_exec.assert_called_once_with('dd',
'if=/dev/zero',
'of=/dev/null',
'count=%s' % units.Gi,
'bs=3M',
'iflag=count_bytes',
'conv=fdatasync,sparse',
run_as_root=True)
def test_copy_volume_dd_with_ionice(self, mock_conf, mock_exec,
mock_support, mock_count, mock_cg):
mock_conf.volume_copy_bps_limit = 10
output = volume_utils.copy_volume('/dev/zero',
'/dev/null',
1024,
1,
sync=True,
execute=utils.execute,
ionice='-c3')
self.assertIsNone(output)
mock_exec.assert_called_once_with('ionice',
'-c3',
'dd',
'if=/dev/zero',
'of=/dev/null',
'count=5678',
'bs=1234',
'conv=fdatasync',
run_as_root=True)
def clear_volume(self, volume):
"""unprovision old volumes to prevent data leaking between users."""
if self.configuration.volume_clear == 'none':
return
vol_path = self.local_path(volume)
size_in_g = volume.get('size', volume.get('volume_size', None))
if size_in_g is None:
LOG.warning(
_("Size for volume: %s not found, "
"skipping secure delete.") % volume['id'])
return
size_in_m = self.configuration.volume_clear_size
LOG.info(_("Performing secure delete on volume: %s") % volume['id'])
if self.configuration.volume_clear == 'zero':
if size_in_m == 0:
return volutils.copy_volume('/dev/zero',
vol_path,
size_in_g * 1024,
sync=True,
execute=self._execute)
else:
clear_cmd = ['shred', '-n0', '-z', '-s%dMiB' % size_in_m]
elif self.configuration.volume_clear == 'shred':
clear_cmd = ['shred', '-n3']
if size_in_m:
clear_cmd.append('-s%dMiB' % size_in_m)
else:
LOG.error(_("Error unrecognized volume_clear option: %s"),
self.configuration.volume_clear)
return
clear_cmd.append(vol_path)
self._execute(*clear_cmd, run_as_root=True)
def copy_volume_data(self, context, src_vol, dest_vol, remote=None):
"""Copy data from src_vol to dest_vol."""
LOG.debug(
_('copy_data_between_volumes %(src)s -> %(dest)s.') % {
'src': src_vol['name'],
'dest': dest_vol['name']
})
properties = utils.brick_get_connector_properties()
dest_remote = True if remote in ['dest', 'both'] else False
dest_orig_status = dest_vol['status']
try:
dest_attach_info = self._attach_volume(context,
dest_vol,
properties,
remote=dest_remote)
except Exception:
with excutils.save_and_reraise_exception():
msg = _("Failed to attach volume %(vol)s")
LOG.error(msg % {'vol': dest_vol['id']})
self.db.volume_update(context, dest_vol['id'],
{'status': dest_orig_status})
src_remote = True if remote in ['src', 'both'] else False
src_orig_status = src_vol['status']
try:
src_attach_info = self._attach_volume(context,
src_vol,
properties,
remote=src_remote)
except Exception:
with excutils.save_and_reraise_exception():
msg = _("Failed to attach volume %(vol)s")
LOG.error(msg % {'vol': src_vol['id']})
self.db.volume_update(context, src_vol['id'],
{'status': src_orig_status})
self._detach_volume(context,
dest_attach_info,
dest_vol,
properties,
force=True,
remote=dest_remote)
try:
size_in_mb = int(src_vol['size']) * 1024 # vol size is in GB
volume_utils.copy_volume(src_attach_info['device']['path'],
dest_attach_info['device']['path'],
size_in_mb,
self.configuration.volume_dd_blocksize)
copy_error = False
except Exception:
with excutils.save_and_reraise_exception():
msg = _("Failed to copy volume %(src)s to %(dest)d")
LOG.error(msg % {'src': src_vol['id'], 'dest': dest_vol['id']})
copy_error = True
finally:
self._detach_volume(context,
dest_attach_info,
dest_vol,
properties,
force=copy_error,
remote=dest_remote)
self._detach_volume(context,
src_attach_info,
src_vol,
properties,
force=copy_error,
remote=src_remote)
def test_copy_volume_handle_transfer(self, mock_open, mock_transfer):
handle = io.RawIOBase()
output = volume_utils.copy_volume('/foo/bar', handle, 1024, 1)
self.assertIsNone(output)
mock_transfer.assert_called_once_with(mock.ANY, mock.ANY, 1073741824,
mock.ANY)
def test_copy_volume_handles(self, mock_copy):
handle1 = io.RawIOBase()
handle2 = io.RawIOBase()
output = volume_utils.copy_volume(handle1, handle2, 1024, 1)
self.assertIsNone(output)
mock_copy.assert_called_once_with(handle1, handle2, 1024)
def fetch_to_volume_format(context, image_service,
image_id, dest, volume_format, blocksize,
user_id=None, project_id=None, size=None,
run_as_root=True):
qemu_img = True
image_meta = image_service.show(context, image_id)
# NOTE(avishay): I'm not crazy about creating temp files which may be
# large and cause disk full errors which would confuse users.
# Unfortunately it seems that you can't pipe to 'qemu-img convert' because
# it seeks. Maybe we can think of something for a future version.
with temporary_file() as tmp:
has_meta = False if not image_meta else True
try:
format_raw = True if image_meta['disk_format'] == 'raw' else False
except TypeError:
format_raw = False
data = get_qemu_data(image_id, has_meta, format_raw,
tmp, run_as_root)
if data is None:
qemu_img = False
tmp_images = TemporaryImages.for_image_service(image_service)
tmp_image = tmp_images.get(context, image_id)
if tmp_image:
tmp = tmp_image
else:
fetch(context, image_service, image_id, tmp, user_id, project_id)
if is_xenserver_format(image_meta):
replace_xenserver_image_with_coalesced_vhd(tmp)
if not qemu_img:
# qemu-img is not installed but we do have a RAW image. As a
# result we only need to copy the image to the destination and then
# return.
LOG.debug('Copying image from %(tmp)s to volume %(dest)s - '
'size: %(size)s', {'tmp': tmp, 'dest': dest,
'size': image_meta['size']})
image_size_m = math.ceil(float(image_meta['size']) / units.Mi)
volume_utils.copy_volume(tmp, dest, image_size_m, blocksize)
return
data = qemu_img_info(tmp, run_as_root=run_as_root)
virt_size = int(math.ceil(float(data.virtual_size) / units.Gi))
# NOTE(xqueralt): If the image virtual size doesn't fit in the
# requested volume there is no point on resizing it because it will
# generate an unusable image.
if size is not None and virt_size > size:
params = {'image_size': virt_size, 'volume_size': size}
reason = _("Size is %(image_size)dGB and doesn't fit in a "
"volume of size %(volume_size)dGB.") % params
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_id)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file, })
# NOTE(e0ne): check for free space in destination directory before
# image convertion.
check_available_space(dest, virt_size, image_id)
# NOTE(jdg): I'm using qemu-img convert to write
# to the volume regardless if it *needs* conversion or not
# TODO(avishay): We can speed this up by checking if the image is raw
# and if so, writing directly to the device. However, we need to keep
# check via 'qemu-img info' that what we copied was in fact a raw
# image and not a different format with a backing file, which may be
# malicious.
LOG.debug("%s was %s, converting to %s ", image_id, fmt, volume_format)
disk_format = fixup_disk_format(image_meta['disk_format'])
convert_image(tmp, dest, volume_format,
src_format=disk_format,
run_as_root=run_as_root)
def migrate_volume(self, ctxt, volume, host, thin=False, mirror_count=0):
"""Optimize the migration if the destination is on the same server.
If the specified host is another back-end on the same server, and
the volume is not attached, we can do the migration locally without
going through iSCSI.
"""
false_ret = (False, None)
if volume['status'] != 'available':
return false_ret
if 'location_info' not in host['capabilities']:
return false_ret
info = host['capabilities']['location_info']
try:
(dest_type, dest_hostname, dest_vg, lvm_type, lvm_mirrors) =\
info.split(':')
lvm_mirrors = int(lvm_mirrors)
except ValueError:
return false_ret
if (dest_type != 'LVMVolumeDriver' or dest_hostname != self.hostname):
return false_ret
if dest_vg == self.vg.vg_name:
message = (_("Refusing to migrate volume ID: %(id)s. Please "
"check your configuration because source and "
"destination are the same Volume Group: %(name)s.") %
{
'id': volume['id'],
'name': self.vg.vg_name
})
LOG.error(message)
raise exception.VolumeBackendAPIException(data=message)
vg_list = volutils.get_all_volume_groups()
try:
next(vg for vg in vg_list if vg['name'] == dest_vg)
except StopIteration:
LOG.error(_LE("Destination Volume Group %s does not exist"),
dest_vg)
return false_ret
helper = utils.get_root_helper()
lvm_conf_file = self.configuration.lvm_conf_file
if lvm_conf_file.lower() == 'none':
lvm_conf_file = None
dest_vg_ref = lvm.LVM(dest_vg,
helper,
lvm_type=lvm_type,
executor=self._execute,
lvm_conf=lvm_conf_file)
self._create_volume(volume['name'], self._sizestr(volume['size']),
lvm_type, lvm_mirrors, dest_vg_ref)
# copy_volume expects sizes in MiB, we store integer GiB
# be sure to convert before passing in
size_in_mb = int(volume['size']) * units.Ki
try:
volutils.copy_volume(self.local_path(volume),
self.local_path(volume, vg=dest_vg),
size_in_mb,
self.configuration.volume_dd_blocksize,
execute=self._execute,
sparse=self._sparse_copy_volume)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Volume migration failed due to "
"exception: %(reason)s."),
{'reason': six.text_type(e)},
resource=volume)
dest_vg_ref.delete(volume)
self._delete_volume(volume)
return (True, None)
def _copy_vdisk_data(self, src_vdisk_name, src_vdisk_id, dest_vdisk_name,
dest_vdisk_id):
"""Copy data from src vdisk to dest vdisk.
To be able to copy data between vdisks, we must ensure that both
vdisks have been mapped to host. If vdisk has not been mapped,
it must be mapped firstly. When data copy completed, vdisk
should be restored to previous mapped or non-mapped status.
"""
LOG.debug('enter: _copy_vdisk_data: %(src)s -> %(dest)s.', {
'src': src_vdisk_name,
'dest': dest_vdisk_name
})
connector = utils.brick_get_connector_properties()
(src_map, src_lun_id) = self._is_vdisk_map(src_vdisk_name, connector)
(dest_map, dest_lun_id) = self._is_vdisk_map(dest_vdisk_name,
connector)
src_map_device = None
src_properties = None
dest_map_device = None
dest_properties = None
try:
if not src_map:
src_lun_id = self._map_vdisk_to_host(src_vdisk_name, connector)
if not dest_map:
dest_lun_id = self._map_vdisk_to_host(dest_vdisk_name,
connector)
src_properties = self._get_vdisk_map_properties(
connector, src_lun_id, src_vdisk_name, src_vdisk_id,
self._get_vdisk_params(None))
src_map_device = self._scan_device(src_properties)
dest_properties = self._get_vdisk_map_properties(
connector, dest_lun_id, dest_vdisk_name, dest_vdisk_id,
self._get_vdisk_params(None))
dest_map_device = self._scan_device(dest_properties)
src_vdisk_attr = self._get_vdisk_attributes(src_vdisk_name)
# vdisk capacity is bytes, translate into MB
size_in_mb = int(src_vdisk_attr['capacity']) / units.Mi
volume_utils.copy_volume(src_map_device['path'],
dest_map_device['path'], size_in_mb,
self.configuration.volume_dd_blocksize)
except Exception:
with excutils.save_and_reraise_exception():
LOG.error('Failed to copy %(src)s to %(dest)s.', {
'src': src_vdisk_name,
'dest': dest_vdisk_name
})
finally:
if not dest_map:
self._unmap_vdisk_from_host(dest_vdisk_name, connector)
self._remove_device(dest_properties, dest_map_device)
if not src_map:
self._unmap_vdisk_from_host(src_vdisk_name, connector)
self._remove_device(src_properties, src_map_device)
LOG.debug('leave: _copy_vdisk_data: %(src)s -> %(dest)s.', {
'src': src_vdisk_name,
'dest': dest_vdisk_name
})
def create_volume_from_snapshot(self, volume, snapshot):
"""Create volume from snapshot.
- search for snapshot and retention_policy
- create a view from snapshot and attach view
- create a volume and attach volume
- copy data from attached view to attached volume
- detach volume and view and finally delete view
"""
snap_name = self.get_snap_name(snapshot.id)
view_name = self.get_view_name(volume.id)
vol_name = self.get_volume_name(volume.id)
cview = src_attach_info = dest_attach_info = None
rpolicy = self.get_policy()
properties = utils.brick_get_connector_properties()
LOG.debug("Searching for snapshot: %s in K2.", snap_name)
snap_rs = self.client.search("snapshots", short_name=snap_name)
if hasattr(snap_rs, 'hits') and snap_rs.total != 0:
snap = snap_rs.hits[0]
LOG.debug("Creating a view: %(view)s from snapshot: %(snap)s",
{'view': view_name, 'snap': snap_name})
try:
cview = self.client.new("snapshots",
short_name=view_name,
source=snap, retention_policy=rpolicy,
is_exposable=True).save()
except Exception as ex:
LOG.exception("Creating a view: %(view)s from snapshot: "
"%(snap)s failed", {"view": view_name,
"snap": snap_name})
raise exception.KaminarioCinderDriverException(reason=ex)
else:
msg = _("Snapshot: %s search failed in K2.") % snap_name
LOG.error(msg)
raise exception.KaminarioCinderDriverException(reason=msg)
try:
conn = self.initialize_connection(cview, properties)
src_attach_info = self._connect_device(conn)
self.create_volume(volume)
conn = self.initialize_connection(volume, properties)
dest_attach_info = self._connect_device(conn)
vol_utils.copy_volume(src_attach_info['device']['path'],
dest_attach_info['device']['path'],
snapshot.volume.size * units.Ki,
self.configuration.volume_dd_blocksize,
sparse=True)
self._kaminario_disconnect_volume(src_attach_info,
dest_attach_info)
self.terminate_connection(volume, properties)
self.terminate_connection(cview, properties)
cview.delete()
except Exception as ex:
self._kaminario_disconnect_volume(src_attach_info,
dest_attach_info)
self.terminate_connection(cview, properties)
self.terminate_connection(volume, properties)
cview.delete()
self.delete_volume(volume)
LOG.exception("Copy to volume: %(vol)s from view: %(view)s "
"failed", {"vol": vol_name, "view": view_name})
raise exception.KaminarioCinderDriverException(reason=ex)
def fetch_to_volume_format(context,
image_service,
image_id,
dest,
volume_format,
blocksize,
user_id=None,
project_id=None,
size=None):
if (CONF.image_conversion_dir
and not os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
qemu_img = True
image_meta = image_service.show(context, image_id)
# NOTE(avishay): I'm not crazy about creating temp files which may be
# large and cause disk full errors which would confuse users.
# Unfortunately it seems that you can't pipe to 'qemu-img convert' because
# it seeks. Maybe we can think of something for a future version.
with temporary_file() as tmp:
# We may be on a system that doesn't have qemu-img installed. That
# is ok if we are working with a RAW image. This logic checks to see
# if qemu-img is installed. If not we make sure the image is RAW and
# throw an exception if not. Otherwise we stop before needing
# qemu-img. Systems with qemu-img will always progress through the
# whole function.
try:
# Use the empty tmp file to make sure qemu_img_info works.
qemu_img_info(tmp)
except processutils.ProcessExecutionError:
qemu_img = False
if image_meta:
if image_meta['disk_format'] != 'raw':
raise exception.ImageUnacceptable(
reason=_("qemu-img is not installed and image is of "
"type %s. Only RAW images can be used if "
"qemu-img is not installed.") %
image_meta['disk_format'],
image_id=image_id)
else:
raise exception.ImageUnacceptable(reason=_(
"qemu-img is not installed and the disk "
"format is not specified. Only RAW images "
"can be used if qemu-img is not installed."),
image_id=image_id)
fetch(context, image_service, image_id, tmp, user_id, project_id)
if is_xenserver_image(context, image_service, image_id):
replace_xenserver_image_with_coalesced_vhd(tmp)
if not qemu_img:
# qemu-img is not installed but we do have a RAW image. As a
# result we only need to copy the image to the destination and then
# return.
LOG.debug('Copying image from %(tmp)s to volume %(dest)s - '
'size: %(size)s' % {
'tmp': tmp,
'dest': dest,
'size': image_meta['size']
})
volume_utils.copy_volume(tmp, dest, image_meta['size'], blocksize)
return
data = qemu_img_info(tmp)
virt_size = data.virtual_size / units.Gi
# NOTE(xqueralt): If the image virtual size doesn't fit in the
# requested volume there is no point on resizing it because it will
# generate an unusable image.
if size is not None and virt_size > size:
params = {'image_size': virt_size, 'volume_size': size}
reason = _("Size is %(image_size)dGB and doesn't fit in a "
"volume of size %(volume_size)dGB.") % params
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."), image_id=image_id)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {
'fmt': fmt,
'backing_file': backing_file,
})
# NOTE(jdg): I'm using qemu-img convert to write
# to the volume regardless if it *needs* conversion or not
# TODO(avishay): We can speed this up by checking if the image is raw
# and if so, writing directly to the device. However, we need to keep
# check via 'qemu-img info' that what we copied was in fact a raw
# image and not a different format with a backing file, which may be
# malicious.
LOG.debug("%s was %s, converting to %s " %
(image_id, fmt, volume_format))
convert_image(tmp,
dest,
volume_format,
bps_limit=CONF.volume_copy_bps_limit)
data = qemu_img_info(dest)
if data.file_format != volume_format:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(vol_format)s, but format is "
"now %(file_format)s") % {
'vol_format': volume_format,
'file_format': data.file_format
})
