def run_iscsiadm(self, sl_vol):
"""
Run `iscsiadm` command on SoftLayer iSCSI target
to fetch IQN and other details of the target.
"""
# modify the iscsid.conf
# first remove and then insert, makes sure we always will have new
# values placed, even if there isn't any existing values.
utils.execute('sed', '-i', '-e',
"/discovery.sendtargets.auth.username/d", '-e',
"/discovery.sendtargets.auth.password/d",
'/etc/iscsi/iscsid.conf',
run_as_root=True)
utils.execute('sed', '-i', '-e',
"1idiscovery.sendtargets.auth.username = %s" % sl_vol[
'username'], '-e',
"1idiscovery.sendtargets.auth.password = %s" % sl_vol[
'password'],
'/etc/iscsi/iscsid.conf',
run_as_root=True)
(out, err) = utils.execute(
'iscsiadm', '-m', 'discovery', '-t', 'st', '-p',
sl_vol['serviceResourceBackendIpAddress'],
'-o', 'new', run_as_root=True)
if err and len(err) != 0:
raise exception.VolumeBackendAPIException(
data="Error while 'discovery' on iSCSI details. %s" % err)
return out
def _verify_rtstool(self):
try:
# This call doesn't need locking
utils.execute('cinder-rtstool', 'verify')
except (OSError, putils.ProcessExecutionError):
LOG.error(_LE('cinder-rtstool is not installed correctly'))
raise
def clear_volume(volume_size, volume_path, volume_clear=None, volume_clear_size=None):
"""Unprovision old volumes to prevent data leaking between users."""
if volume_clear is None:
volume_clear = CONF.volume_clear
if volume_clear_size is None:
volume_clear_size = CONF.volume_clear_size
if volume_clear_size == 0:
volume_clear_size = volume_size
LOG.info(_("Performing secure delete on volume: %s") % volume_path)
if volume_clear == "zero":
return copy_volume(
"/dev/zero", volume_path, volume_clear_size, CONF.volume_dd_blocksize, sync=True, execute=utils.execute
)
elif volume_clear == "shred":
clear_cmd = ["shred", "-n3"]
if volume_clear_size:
clear_cmd.append("-s%dMiB" % volume_clear_size)
else:
raise exception.InvalidConfigurationValue(option="volume_clear", value=volume_clear)
clear_cmd.append(volume_path)
utils.execute(*clear_cmd, run_as_root=True)
def check_for_setup_error(self):
"""Check that the driver is working and can communicate.
Invoke a web services API to make sure we can talk to the server.
Also perform the datacenter value verification.
"""
try:
LOG.debug("Checking if sed is accessible as root")
utils.execute(
"sed",
"-e",
"/discovery.sendtargets.auth.username/d",
"-e",
"/discovery.sendtargets.auth.password/d",
"/etc/iscsi/iscsid.conf",
run_as_root=True,
)
except proc_utils.ProcessExecutionError as ex:
LOG.error(
_(
"Uable to execute 'sed' command on "
"/etc/iscsi/iscsid.conf make sure you have"
"'sed' entry in cinder's rootwrap.conf "
"or check if /etc/iscsi/iscsid.conf exists"
)
)
raise ex
return self.vol_mgr.check_dc()
def test_fetch_verify_image_without_file_format(self):
TEST_RETURN = (
"image: qemu.qcow2\n"
"virtual_size: 50M (52428800 bytes)\n"
"cluster_size: 65536\n"
"disk_size: 196K (200704 bytes)\n"
"Snapshot list:\n"
"ID TAG VM SIZE DATE VM CLOCK\n"
"1 snap1 1.7G 2011-10-04 19:04:00 32:06:34.974"
)
fake_image_service = FakeImageService()
mox = self._mox
mox.StubOutWithMock(image_utils, "fetch")
mox.StubOutWithMock(utils, "execute")
image_utils.fetch(context, fake_image_service, self.TEST_IMAGE_ID, self.TEST_DEV_PATH, None, None)
utils.execute("env", "LC_ALL=C", "LANG=C", "qemu-img", "info", self.TEST_DEV_PATH, run_as_root=True).AndReturn(
(TEST_RETURN, "ignored")
)
mox.ReplayAll()
self.assertRaises(
exception.ImageUnacceptable,
image_utils.fetch_verify_image,
context,
fake_image_service,
self.TEST_IMAGE_ID,
self.TEST_DEV_PATH,
)
def _cleanup_device_hardlink(self,
hardlink_path,
volume_path,
volume_id):
"""Remove the hardlink for the volume block device.
:param hardlink_path: hardlink to the volume block device
:param volume_path: real path of the backup/restore device
:param volume_id: Volume id for backup or as restore target
"""
try:
utils.execute('rm',
'-f',
hardlink_path,
run_as_root=True)
except processutils.ProcessExecutionError as e:
err = (_('backup: %(vol_id)s Failed to remove backup hardlink'
' from %(vpath)s to %(bpath)s.\n'
'stdout: %(out)s\n stderr: %(err)s')
% {'vol_id': volume_id,
'vpath': volume_path,
'bpath': hardlink_path,
'out': e.stdout,
'err': e.stderr})
LOG.error(err)
def convert_image(source, dest, out_format, bps_limit=None):
"""Convert image to other format."""
start_time = timeutils.utcnow()
cmd = ('qemu-img', 'convert', '-O', out_format, source, dest)
cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit)
if cgcmd:
cmd = tuple(cgcmd) + cmd
cmd += ('-t', 'none') # required to enable ratelimit by blkio cgroup
utils.execute(*cmd, run_as_root=True)
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
# NOTE(jdg): use a default of 1, mostly for unit test, but in
# some incredible event this is 0 (cirros image?) don't barf
if duration < 1:
duration = 1
fsz_mb = os.stat(source).st_size / units.Mi
mbps = (fsz_mb / duration)
msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
"duration %(duration).2f sec, destination %(dest)s")
LOG.debug(msg % {"src": source,
"sz": fsz_mb,
"duration": duration,
"dest": dest})
msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s")
LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def _make_link(volume_path, backup_path, vol_id):
"""Create a hard link for the volume block device.
The IBM TSM client performs an image backup on a block device.
The name of the block device is the backup prefix plus the backup id
:param volume_path: real device path name for volume
:param backup_path: path name TSM will use as volume to backup
:param vol_id: id of volume to backup (for reporting)
:raises: InvalidBackup
"""
try:
utils.execute('ln', volume_path, backup_path,
run_as_root=True,
check_exit_code=True)
except processutils.ProcessExecutionError as exc:
err = (_('backup: %(vol_id)s failed to create device hardlink '
'from %(vpath)s to %(bpath)s.\n'
'stdout: %(out)s\n stderr: %(err)s')
% {'vol_id': vol_id,
'vpath': volume_path,
'bpath': backup_path,
'out': exc.stdout,
'err': exc.stderr})
LOG.error(err)
raise exception.InvalidBackup(reason=err)
def remove_iscsi_target(self, tid, lun, vol_id, vol_name, **kwargs):
LOG.info(_LI('Removing iscsi_target for: %s') % vol_id)
vol_uuid_file = vol_name
volume_path = os.path.join(self.volumes_dir, vol_uuid_file)
if not os.path.exists(volume_path):
LOG.warning(_LW('Volume path %s does not exist, '
'nothing to remove.') % volume_path)
return
if os.path.isfile(volume_path):
iqn = '%s%s' % (self.iscsi_target_prefix,
vol_uuid_file)
else:
raise exception.ISCSITargetRemoveFailed(volume_id=vol_id)
try:
# NOTE(vish): --force is a workaround for bug:
# https://bugs.launchpad.net/cinder/+bug/1159948
utils.execute('tgt-admin',
'--force',
'--delete',
iqn,
run_as_root=True)
except putils.ProcessExecutionError as e:
LOG.error(_LE("Failed to remove iscsi target for volume "
"id:%(vol_id)s: %(e)s")
% {'vol_id': vol_id, 'e': e})
raise exception.ISCSITargetRemoveFailed(volume_id=vol_id)
# NOTE(jdg): There's a bug in some versions of tgt that
# will sometimes fail silently when using the force flag
# https://bugs.launchpad.net/ubuntu/+source/tgt/+bug/1305343
# For now work-around by checking if the target was deleted,
# if it wasn't, try again without the force.
# This will NOT do any good for the case of mutliple sessions
# which the force was aded for but it will however address
# the cases pointed out in bug:
# https://bugs.launchpad.net/cinder/+bug/1304122
if self._get_target(iqn):
try:
LOG.warning(_LW('Silent failure of target removal '
'detected, retry....'))
utils.execute('tgt-admin',
'--delete',
iqn,
run_as_root=True)
except putils.ProcessExecutionError as e:
LOG.error(_LE("Failed to remove iscsi target for volume "
"id:%(vol_id)s: %(e)s")
% {'vol_id': vol_id, 'e': e})
raise exception.ISCSITargetRemoveFailed(volume_id=vol_id)
# NOTE(jdg): This *should* be there still but incase
# it's not we don't care, so just ignore it if was
# somehow deleted between entry of this method
# and here
if os.path.exists(volume_path):
os.unlink(volume_path)
else:
LOG.debug('Volume path %s not found at end, '
'of remove_iscsi_target.' % volume_path)
def test_fetch_verify_image_with_backing_file(self):
TEST_RETURN = "image: qemu.qcow2\n"\
"backing_file: qemu.qcow2 (actual path: qemu.qcow2)\n"\
"file_format: qcow2\n"\
"virtual_size: 50M (52428800 bytes)\n"\
"cluster_size: 65536\n"\
"disk_size: 196K (200704 bytes)\n"\
"Snapshot list:\n"\
"ID TAG VM SIZE DATE VM CLOCK\n"\
"1 snap1 1.7G 2011-10-04 19:04:00 32:06:34.974"
fake_image_service = FakeImageService()
mox = self._mox
mox.StubOutWithMock(image_utils, 'fetch')
mox.StubOutWithMock(utils, 'execute')
image_utils.fetch(context, fake_image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH, None, None)
utils.execute(
'env', 'LC_ALL=C', 'LANG=C', 'qemu-img', 'info',
self.TEST_DEV_PATH, run_as_root=True).AndReturn(
(TEST_RETURN, 'ignored')
)
mox.ReplayAll()
self.assertRaises(exception.ImageUnacceptable,
image_utils.fetch_verify_image,
context, fake_image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
def create_volume(self, volume):
LOG.debug("Create volume")
host = volutils.extract_host(volume["host"], "host")
try:
self.restapi.post(
"nbd" + self._get_remote_url(host),
{
"objectPath": self.bucket_path + "/" + volume["name"],
"volSizeMB": int(volume["size"]) * units.Ki,
"blockSize": self.blocksize,
"chunkSize": self.chunksize,
},
)
number = self._get_nbd_number(volume)
cinder_utils.execute(
"ln",
"--symbolic",
"--force",
"/dev/nbd" + six.text_type(number),
self._get_symlink_path(number),
run_as_root=True,
check_exit_code=True,
)
except exception.VolumeBackendAPIException:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Error creating volume"))
def _disk_vmdk_to_qcow2(path):
"""Converts a vmdk disk to qcow2."""
path_qcow2 = path + '_qcow2'
utils.execute('qemu-img', 'convert', '-f', 'vmdk',
'-O', 'qcow2','-c', path, path_qcow2)
utils.execute('mv', path_qcow2, path)
return path
def create_empty_file(filename):
if not os.path.exists(filename):
try:
utils.execute("touch", filename)
except putils.ProcessExecutionError as ex:
msg = output_err(631, file=filename, ret=ex.exit_code, err=ex.stderr)
raise exception.HBSDError(message=msg)
def initialize_connection(self, volume, connector):
volume_iqn = volume['provider_location'].split(' ')[1]
(auth_method, auth_user, auth_pass) = \
volume['provider_auth'].split(' ', 3)
# Add initiator iqns to target ACL
try:
utils.execute('cinder-rtstool', 'add-initiator',
volume_iqn,
auth_user,
auth_pass,
connector['initiator'],
run_as_root=True)
except putils.ProcessExecutionError:
LOG.error(_LE("Failed to add initiator iqn %s to target") %
connector['initiator'])
raise exception.ISCSITargetAttachFailed(
volume_id=volume['id'])
# We make changes persistent
self._persist_configuration(volume['id'])
iscsi_properties = self._get_iscsi_properties(volume,
connector.get(
'multipath'))
return {
'driver_volume_type': self.iscsi_protocol,
'data': iscsi_properties
}
def test_upload_volume(self, bps_limit=0):
image_meta = {'id': 1, 'disk_format': 'qcow2'}
TEST_RET = "image: qemu.qcow2\n"\
"file_format: qcow2 \n"\
"virtual_size: 50M (52428800 bytes)\n"\
"cluster_size: 65536\n"\
"disk_size: 196K (200704 bytes)"
if bps_limit:
CONF.set_override('volume_copy_bps_limit', bps_limit)
prefix = ('cgexec', '-g', 'blkio:test')
postfix = ('-t', 'none')
else:
prefix = postfix = ()
cmd = prefix + ('qemu-img', 'convert', '-O', 'qcow2',
mox.IgnoreArg(), mox.IgnoreArg()) + postfix
m = self._mox
m.StubOutWithMock(utils, 'execute')
m.StubOutWithMock(volume_utils, 'setup_blkio_cgroup')
volume_utils.setup_blkio_cgroup(mox.IgnoreArg(), mox.IgnoreArg(),
bps_limit).AndReturn(prefix)
utils.execute(*cmd, run_as_root=True)
utils.execute(
'env', 'LC_ALL=C', 'qemu-img', 'info',
mox.IgnoreArg(), run_as_root=True).AndReturn(
(TEST_RET, 'ignored')
)
m.ReplayAll()
image_utils.upload_volume(context, FakeImageService(),
image_meta, '/dev/loop1')
m.VerifyAll()
def test_upload_volume_on_error(self):
image_meta = {'id': 1, 'disk_format': 'qcow2'}
TEST_RET = "image: qemu.vhd\n"\
"file_format: vhd \n"\
"virtual_size: 50M (52428800 bytes)\n"\
"cluster_size: 65536\n"\
"disk_size: 196K (200704 bytes)"
m = self._mox
m.StubOutWithMock(utils, 'execute')
utils.execute('qemu-img', 'convert', '-O', 'qcow2',
mox.IgnoreArg(), mox.IgnoreArg(), run_as_root=True)
utils.execute(
'env', 'LC_ALL=C', 'qemu-img', 'info',
mox.IgnoreArg(), run_as_root=True).AndReturn(
(TEST_RET, 'ignored')
)
m.ReplayAll()
self.assertRaises(exception.ImageUnacceptable,
image_utils.upload_volume,
context, FakeImageService(),
image_meta, '/dev/loop1')
m.VerifyAll()
def _limit_bps(self, rw, dev, bps):
try:
utils.execute('cgset', '-r', 'blkio.throttle.%s_bps_device=%s %d'
% (rw, dev, bps), self.cgroup, run_as_root=True)
except processutils.ProcessExecutionError:
LOG.warning(_LW('Failed to setup blkio cgroup to throttle the '
'device \'%(device)s\'.'), {'device': dev})
def delete_volume(self, volume):
"""
Removes the data from volume and returns the volume to pool.
:param volume: OpenStack Volume Object.
"""
if self.configuration.sl_pool_volume_clear not in ("zero", "shred", "none"):
raise exception.InvalidConfigurationValue(
option="volume_clear", value=self.configuration.sl_pool_volume_clear
)
sl_vol = self.meta_mgr.deserialize(volume["id"])
connection = self.vol_mgr.get_iscsi_properties(sl_vol)
attach_info = self._attch(connection)
size_in_mb = 1024 * volume["size"]
try:
if self.configuration.sl_pool_volume_clear == "zero":
LOG.info("zeroing out volume")
volume_utils.copy_volume("/dev/zero", attach_info["device"]["path"], size_in_mb)
elif self.configuration.sl_pool_volume_clear == "shred":
LOG.info("Shredding volume")
utils.execute("shred", "-n3", "-s%dMiB" % size_in_mb, attach_info["device"]["path"], run_as_root=True)
except proc_utils.ProcessExecutionError as ex:
LOG.error(_("Error while swiping out data. %s" % ex))
raise
finally:
self._detach_volume(attach_info)
self.meta_mgr.delete_all(volume["id"])
def _disk_qcow2_to_vmdk(path):
"""Converts a qcow2 disk to vmdk."""
path_vmdk = path + '_vmdk'
utils.execute('qemu-img', 'convert', '-f', 'qcow2',
'-O', 'vmdk', path, path_vmdk)
utils.execute('mv', path_vmdk, path)
return path
def convert_image(source, dest, out_format, bps_limit=None):
"""Convert image to other format."""
start_time = timeutils.utcnow()
# Always set -t none. First it is needed for cgroup io/limiting
# and it is needed to ensure that all data hit the device before
# it gets unmapped remotely from the host
cmd = ('qemu-img', 'convert',
'-t', 'none',
'-O', out_format, source, dest)
cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit)
if cgcmd:
cmd = tuple(cgcmd) + cmd
utils.execute(*cmd, run_as_root=True)
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
# NOTE(jdg): use a default of 1, mostly for unit test, but in
# some incredible event this is 0 (cirros image?) don't barf
if duration < 1:
duration = 1
fsz_mb = os.stat(source).st_size / units.Mi
mbps = (fsz_mb / duration)
msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
"duration %(duration).2f sec, destination %(dest)s")
LOG.debug(msg % {"src": source,
"sz": fsz_mb,
"duration": duration,
"dest": dest})
msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s")
LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def update_config_file(self, name, tid, path, config_auth):
conf_file = self.iet_conf
vol_id = name.split(':')[1]
# If config file does not exist, create a blank conf file and
# add configuration for the volume on the new file.
if not os.path.exists(conf_file):
try:
utils.execute("truncate", conf_file, "--size=0",
run_as_root=True)
except putils.ProcessExecutionError:
LOG.exception(_LE("Failed to create %(conf)s for volume "
"id:%(vol_id)s"),
{'conf': conf_file, 'vol_id': vol_id})
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
try:
volume_conf = """
Target %s
%s
Lun 0 Path=%s,Type=%s
""" % (name, config_auth, path, self._iotype(path))
with utils.temporary_chown(conf_file):
with open(conf_file, 'a+') as f:
f.write(volume_conf)
except Exception:
LOG.exception(_LE("Failed to update %(conf)s for volume "
"id:%(vol_id)s"),
{'conf': conf_file, 'vol_id': vol_id})
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
def initialize_connection(self, volume, connector):
volume_iqn = volume['provider_location'].split(' ')[1]
(auth_method, auth_user, auth_pass) = \
volume['provider_auth'].split(' ', 3)
# Add initiator iqns to target ACL
try:
utils.execute('cinder-rtstool', 'add-initiator',
volume_iqn,
auth_user,
auth_pass,
connector['initiator'],
run_as_root=True)
except putils.ProcessExecutionError:
LOG.error(_LE("Failed to add initiator iqn %s to target") %
connector['initiator'])
raise exception.ISCSITargetAttachFailed(
volume_id=volume['id'])
iscsi_properties = self._get_iscsi_properties(volume)
# FIXME(jdg): For LIO the target_lun is 0, other than that all data
# is the same as it is for tgtadm, just modify it here
iscsi_properties['target_lun'] = 0
return {
'driver_volume_type': 'iscsi',
'data': iscsi_properties
}
def clear_volume(volume_size, volume_path, volume_clear=None,
volume_clear_size=None, volume_clear_ionice=None):
"""Unprovision old volumes to prevent data leaking between users."""
if volume_clear is None:
volume_clear = CONF.volume_clear
if volume_clear_size is None:
volume_clear_size = CONF.volume_clear_size
if volume_clear_size == 0:
volume_clear_size = volume_size
if volume_clear_ionice is None:
volume_clear_ionice = CONF.volume_clear_ionice
LOG.info(_("Performing secure delete on volume: %s") % volume_path)
if volume_clear == 'zero':
return copy_volume('/dev/zero', volume_path, volume_clear_size,
CONF.volume_dd_blocksize,
sync=True, execute=utils.execute,
ionice=volume_clear_ionice)
elif volume_clear == 'shred':
clear_cmd = ['shred', '-n3']
if volume_clear_size:
clear_cmd.append('-s%dMiB' % volume_clear_size)
else:
raise exception.InvalidConfigurationValue(
option='volume_clear',
value=volume_clear)
clear_cmd.append(volume_path)
utils.execute(*clear_cmd, run_as_root=True)
def test_fetch_to_raw_on_error_backing_file(self):
TEST_RET = "image: qemu.qcow2\n"\
"backing_file: qemu.qcow2 (actual path: qemu.qcow2)\n"\
"file_format: qcow2 \n"\
"virtual_size: 50M (52428800 bytes)\n"\
"cluster_size: 65536\n"\
"disk_size: 196K (200704 bytes)"
fake_image_service = FakeImageService()
mox = self._mox
mox.StubOutWithMock(image_utils, 'create_temporary_file')
mox.StubOutWithMock(utils, 'execute')
mox.StubOutWithMock(image_utils, 'fetch')
image_utils.create_temporary_file().AndReturn(self.TEST_DEV_PATH)
image_utils.fetch(context, fake_image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH, None, None)
utils.execute(
'env', 'LC_ALL=C', 'LANG=C', 'qemu-img', 'info',
self.TEST_DEV_PATH, run_as_root=True).AndReturn(
(TEST_RET, 'ignored')
)
mox.ReplayAll()
self.assertRaises(exception.ImageUnacceptable,
image_utils.fetch_to_raw,
context, fake_image_service,
self.TEST_IMAGE_ID, self.TEST_DEV_PATH)
def test_upload_volume_on_error(self):
image_meta = {"id": 1, "disk_format": "qcow2"}
TEST_RET = (
"image: qemu.vhd\n"
"file_format: vhd \n"
"virtual_size: 50M (52428800 bytes)\n"
"cluster_size: 65536\n"
"disk_size: 196K (200704 bytes)"
)
m = self._mox
m.StubOutWithMock(utils, "execute")
utils.execute("qemu-img", "convert", "-O", "qcow2", mox.IgnoreArg(), mox.IgnoreArg(), run_as_root=True)
utils.execute("env", "LC_ALL=C", "LANG=C", "qemu-img", "info", mox.IgnoreArg(), run_as_root=True).AndReturn(
(TEST_RET, "ignored")
)
m.ReplayAll()
self.assertRaises(
exception.ImageUnacceptable,
image_utils.upload_volume,
context,
FakeImageService(),
image_meta,
"/dev/loop1",
)
m.VerifyAll()
def _limit_bps(self, rw, dev, bps):
try:
utils.execute(
"cgset", "-r", "blkio.throttle.%s_bps_device=%s %d" % (rw, dev, bps), self.cgroup, run_as_root=True
)
except processutils.ProcessExecutionError:
LOG.warning(_LW("Failed to setup blkio cgroup to throttle the " "device '%(device)s'."), {"device": dev})
def test_upload_volume_with_bps_limit(self, mock_stat):
bps_limit = 1048576
image_meta = {"id": 1, "disk_format": "qcow2"}
TEST_RET = (
"image: qemu.qcow2\n"
"file_format: qcow2 \n"
"virtual_size: 50M (52428800 bytes)\n"
"cluster_size: 65536\n"
"disk_size: 196K (200704 bytes)"
)
self.override_config("volume_copy_bps_limit", bps_limit)
prefix = ("cgexec", "-g", "blkio:test")
cmd = prefix + ("qemu-img", "convert", "-O", "qcow2", mox.IgnoreArg(), mox.IgnoreArg())
m = self._mox
m.StubOutWithMock(utils, "execute")
m.StubOutWithMock(volume_utils, "setup_blkio_cgroup")
m.StubOutWithMock(volume_utils, "check_for_odirect_support")
volume_utils.setup_blkio_cgroup(mox.IgnoreArg(), mox.IgnoreArg(), bps_limit).AndReturn(prefix)
utils.execute(*cmd, run_as_root=True)
utils.execute("env", "LC_ALL=C", "qemu-img", "info", mox.IgnoreArg(), run_as_root=True).AndReturn(
(TEST_RET, "ignored")
)
m.ReplayAll()
image_utils.upload_volume(context, FakeImageService(), image_meta, "/dev/loop1")
m.VerifyAll()
def test_clear_volume_shred_not_clear_size(self):
CONF.volume_clear = 'shred'
CONF.volume_clear_size = None
clear_cmd = ['shred', '-n3', "volume_path"]
self.mox.StubOutWithMock(utils, "execute")
utils.execute(*clear_cmd, run_as_root=True)
self.mox.ReplayAll()
volume_utils.clear_volume(1024, "volume_path")
def test_clear_volume_shred(self):
CONF.volume_clear = "shred"
CONF.volume_clear_size = 1
clear_cmd = ["shred", "-n3", "-s1MiB", "volume_path"]
self.mox.StubOutWithMock(utils, "execute")
utils.execute(*clear_cmd, run_as_root=True)
self.mox.ReplayAll()
volume_utils.clear_volume(1024, "volume_path")
def convert_image(source, dest, out_format, bps_limit=None):
"""Convert image to other format."""
cmd = ('qemu-img', 'convert', '-O', out_format, source, dest)
cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit)
if cgcmd:
cmd = tuple(cgcmd) + cmd
cmd += ('-t', 'none') # required to enable ratelimit by blkio cgroup
utils.execute(*cmd, run_as_root=True)
def _delete_target(self, tid):
utils.execute('ietadm',
'--op',
'delete',
'--tid=%s' % tid,
run_as_root=True)
def set_vhd_parent(vhd_path, parentpath):
utils.execute('vhd-util', 'modify', '-n', vhd_path, '-p', parentpath)
def show_target(self, tid, iqn=None):
utils.execute('ietadm',
'--op',
'show',
'--tid=%s' % tid,
run_as_root=True)
def resize_vhd(vhd_path: str, size: int, journal: str) -> None:
utils.execute('vhd-util', 'resize', '-n', vhd_path, '-s', '%d' % size,
'-j', journal)
def extract_targz(archive_name: str, target: str) -> None:
utils.execute('tar', '-xzf', archive_name, '-C', target)
def _attach_volume(self, volume, sdc_ip):
# We need to make sure we even *have* a local path
LOG.info("ScaleIO attach volume in scaleio cinder driver")
volname = self.id_to_base64(volume.id)
cmd = ['drv_cfg']
cmd += ["--query_guid"]
LOG.info("ScaleIO sdc query guid command: " + str(cmd))
try:
(out, err) = utils.execute(*cmd, run_as_root=True)
LOG.info("map volume %s: stdout=%s stderr=%s" % (cmd, out, err))
except processutils.ProcessExecutionError as e:
msg = ("Error querying sdc guid: %s" % (e.stderr))
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
guid = out
msg = ("Current sdc guid: %s" % (guid))
LOG.info(msg)
params = {'guid': guid}
volume_id = self._get_volume_id(volname)
headers = {'content-type': 'application/json'}
request = "https://" + self.server_ip + ":" + self.server_port + "/api/instances/Volume::" + str(
volume_id) + "/action/addMappedSdc"
LOG.info("map volume request: %s" % request)
if (self.verify_server_certificate == 'True'):
verify_cert = self.server_certificate_path
else:
verify_cert = False
r = requests.post(request,
data=json.dumps(params),
headers=headers,
auth=(self.server_username, self.server_token),
verify=verify_cert)
r = self._check_response(r, request)
# LOG.info("map volume response: %s" % r.text)
if (r.status_code != OK_STATUS_CODE):
response = r.json()
error_code = response['errorCode']
if (error_code == VOLUME_ALREADY_MAPPED_ERROR):
msg = (
"Ignoring error mapping volume %s: volume already mapped" %
(volname))
LOG.warning(msg)
else:
msg = ("Error mapping volume %s: %s" %
(volname, response['message']))
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
# convert id to hex
# val = int(volume_id)
# id_in_hex = hex((val + (1 << 64)) % (1 << 64))
# formated_id = id_in_hex.rstrip("L").lstrip("0x") or "0"
formated_id = volume_id
return self.find_volume_path(formated_id)
def set_vhd_parent(vhd_path: str, parentpath: str) -> None:
utils.execute('vhd-util', 'modify', '-n', vhd_path, '-p', parentpath)
def _convert_image(prefix,
source,
dest,
out_format,
out_subformat=None,
src_format=None,
run_as_root=True,
cipher_spec=None,
passphrase_file=None):
"""Convert image to other format."""
# Check whether O_DIRECT is supported and set '-t none' if it is
# This is needed to ensure that all data hit the device before
# it gets unmapped remotely from the host for some backends
# Reference Bug: #1363016
# NOTE(jdg): In the case of file devices qemu does the
# flush properly and more efficiently than would be done
# setting O_DIRECT, so check for that and skip the
# setting for non BLK devs
if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(
source, dest, 'oflag=direct')):
cache_mode = 'none'
else:
# use default
cache_mode = None
cmd = _get_qemu_convert_cmd(source,
dest,
out_format=out_format,
src_format=src_format,
out_subformat=out_subformat,
cache_mode=cache_mode,
prefix=prefix,
cipher_spec=cipher_spec,
passphrase_file=passphrase_file)
start_time = timeutils.utcnow()
utils.execute(*cmd, run_as_root=run_as_root)
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
# NOTE(jdg): use a default of 1, mostly for unit test, but in
# some incredible event this is 0 (cirros image?) don't barf
if duration < 1:
duration = 1
try:
image_size = qemu_img_info(source,
run_as_root=run_as_root).virtual_size
except ValueError as e:
msg = ("The image was successfully converted, but image size "
"is unavailable. src %(src)s, dest %(dest)s. %(error)s")
LOG.info(msg, {"src": source, "dest": dest, "error": e})
return
fsz_mb = image_size / units.Mi
mbps = (fsz_mb / duration)
msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
"duration %(duration).2f sec, destination %(dest)s")
LOG.debug(msg, {
"src": source,
"sz": fsz_mb,
"duration": duration,
"dest": dest
})
msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s"
LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def _convert_image(prefix, source, dest, out_format,
out_subformat=None, src_format=None,
run_as_root=True, cipher_spec=None,
passphrase_file=None, compress=False):
"""Convert image to other format.
:param prefix: command prefix, i.e. cgexec for throttling
:param source: source filename
:param dest: destination filename
:param out_format: output image format of qemu-img
:param out_subformat: output image subformat
:param src_format: source image format
:param run_as_root: run qemu-img as root
:param cipher_spec: encryption details
:param passphrase_file: filename containing luks passphrase
:param compress: compress w/ qemu-img when possible (best effort)
"""
# Check whether O_DIRECT is supported and set '-t none' if it is
# This is needed to ensure that all data hit the device before
# it gets unmapped remotely from the host for some backends
# Reference Bug: #1363016
# NOTE(jdg): In the case of file devices qemu does the
# flush properly and more efficiently than would be done
# setting O_DIRECT, so check for that and skip the
# setting for non BLK devs
if (utils.is_blk_device(dest) and
volume_utils.check_for_odirect_support(source,
dest,
'oflag=direct')):
cache_mode = 'none'
else:
# use default
cache_mode = None
cmd = _get_qemu_convert_cmd(source, dest,
out_format=out_format,
src_format=src_format,
out_subformat=out_subformat,
cache_mode=cache_mode,
prefix=prefix,
cipher_spec=cipher_spec,
passphrase_file=passphrase_file,
compress=compress)
start_time = timeutils.utcnow()
# If there is not enough space on the conversion partition, include
# the partitions's name in the error message.
try:
utils.execute(*cmd, run_as_root=run_as_root)
except processutils.ProcessExecutionError as ex:
if "No space left" in ex.stderr and CONF.image_conversion_dir in dest:
conversion_dir = CONF.image_conversion_dir
while not os.path.ismount(conversion_dir):
conversion_dir = os.path.dirname(conversion_dir)
message = _("Insufficient free space on %(location)s for image "
"conversion.") % {'location': conversion_dir}
LOG.error(message)
raise
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
# NOTE(jdg): use a default of 1, mostly for unit test, but in
# some incredible event this is 0 (cirros image?) don't barf
if duration < 1:
duration = 1
try:
image_size = qemu_img_info(source,
run_as_root=run_as_root).virtual_size
except ValueError as e:
msg = ("The image was successfully converted, but image size "
"is unavailable. src %(src)s, dest %(dest)s. %(error)s")
LOG.info(msg, {"src": source,
"dest": dest,
"error": e})
return
fsz_mb = image_size / units.Mi
mbps = (fsz_mb / duration)
msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
"duration %(duration).2f sec, destination %(dest)s")
LOG.debug(msg, {"src": source,
"sz": fsz_mb,
"duration": duration,
"dest": dest})
msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s"
LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def san_execute(self, *cmd, **kwargs):
if self.run_local:
return utils.execute(*cmd, **kwargs)
else:
check_exit_code = kwargs.pop('check_exit_code', None)
return self._run_ssh(cmd, check_exit_code)
def run_cmd(self, cmd, ip0, user, pw, *args, **kwargs):
"""Run a command on SMU or using SSH
:param cmd: ssc command name
:param ip0: string IP address of controller
:param user: string user authentication for array
:param pw: string password authentication for array
:return: formated string with version information
"""
LOG.debug('Enable ssh: %s',
six.text_type(self.drv_configs['ssh_enabled']))
if self.drv_configs['ssh_enabled'] != 'True':
# Direct connection via ssc
args = (cmd, '--user', user, '--password', pw, ip0) + args
try:
out, err = utils.execute(*args, **kwargs)
LOG.debug(
"command %(cmd)s result: out = %(out)s - err = "
"%(err)s", {
'cmd': cmd,
'out': out,
'err': err
})
return out, err
except putils.ProcessExecutionError as e:
if 'Failed to establish SSC connection' in e.stderr:
LOG.debug("SSC connection error!")
msg = _("Failed to establish SSC connection.")
raise exception.HNASConnError(msg)
elif 'Connection reset' in e.stderr:
LOG.debug("HNAS connection reset!")
msg = _("HNAS has disconnected SSC")
raise exception.HNASConnError(msg)
else:
raise
else:
if self.drv_configs['cluster_admin_ip0'] is None:
# Connect to SMU through SSH and run ssc locally
args = (cmd, 'localhost') + args
else:
args = (cmd, '--smuauth',
self.drv_configs['cluster_admin_ip0']) + args
utils.check_ssh_injection(args)
command = ' '.join(args)
command = command.replace('"', '\\"')
if not self.sshpool:
server = self.drv_configs['mgmt_ip0']
port = int(self.drv_configs['ssh_port'])
username = self.drv_configs['username']
# We only accept private/public key auth
password = ""
privatekey = self.drv_configs['ssh_private_key']
self.sshpool = ssh_utils.SSHPool(server,
port,
None,
username,
password=password,
privatekey=privatekey)
with self.sshpool.item() as ssh:
try:
out, err = putils.ssh_execute(ssh,
command,
check_exit_code=True)
LOG.debug(
"command %(cmd)s result: out = "
"%(out)s - err = %(err)s", {
'cmd': cmd,
'out': out,
'err': err
})
return out, err
except putils.ProcessExecutionError as e:
if 'Failed to establish SSC connection' in e.stderr:
LOG.debug("SSC connection error!")
msg = _("Failed to establish SSC connection.")
raise exception.HNASConnError(msg)
else:
raise putils.ProcessExecutionError
def _change_file_mode(self, filepath):
utils.execute('chmod', '777', filepath)
def coalesce_vhd(vhd_path):
utils.execute(
'vhd-util', 'coalesce', '-n', vhd_path)
def resize_vhd(vhd_path, size, journal):
utils.execute(
'vhd-util', 'resize', '-n', vhd_path, '-s', '%d' % size, '-j', journal)
def remove_iscsi_target(self, tid, lun, vol_id, vol_name, **kwargs):
LOG.info(_LI('Removing iscsi_target for Volume ID: %s'), vol_id)
vol_uuid_file = vol_name
volume_path = os.path.join(self.volumes_dir, vol_uuid_file)
if not os.path.exists(volume_path):
LOG.warning(
_LW('Volume path %s does not exist, '
'nothing to remove.'), volume_path)
return
if os.path.isfile(volume_path):
iqn = '%s%s' % (self.iscsi_target_prefix, vol_uuid_file)
else:
raise exception.ISCSITargetRemoveFailed(volume_id=vol_id)
try:
# NOTE(vish): --force is a workaround for bug:
# https://bugs.launchpad.net/cinder/+bug/1159948
utils.execute('tgt-admin',
'--force',
'--delete',
iqn,
run_as_root=True)
except putils.ProcessExecutionError as e:
non_fatal_errors = ("can't find the target",
"access control rule does not exist")
if any(error in e.stderr for error in non_fatal_errors):
LOG.warning(
_LW("Failed target removal because target or "
"ACL's couldn't be found for iqn: %s."), iqn)
else:
LOG.error(
_LE("Failed to remove iscsi target for Volume "
"ID: %(vol_id)s: %(e)s"), {
'vol_id': vol_id,
'e': e
})
raise exception.ISCSITargetRemoveFailed(volume_id=vol_id)
# NOTE(jdg): There's a bug in some versions of tgt that
# will sometimes fail silently when using the force flag
# https://bugs.launchpad.net/ubuntu/+source/tgt/+bug/1305343
# For now work-around by checking if the target was deleted,
# if it wasn't, try again without the force.
# This will NOT do any good for the case of mutliple sessions
# which the force was aded for but it will however address
# the cases pointed out in bug:
# https://bugs.launchpad.net/cinder/+bug/1304122
if self._get_target(iqn):
try:
LOG.warning(
_LW('Silent failure of target removal '
'detected, retry....'))
utils.execute('tgt-admin', '--delete', iqn, run_as_root=True)
except putils.ProcessExecutionError as e:
LOG.error(
_LE("Failed to remove iscsi target for Volume "
"ID: %(vol_id)s: %(e)s"), {
'vol_id': vol_id,
'e': e
})
raise exception.ISCSITargetRemoveFailed(volume_id=vol_id)
# NOTE(jdg): This *should* be there still but incase
# it's not we don't care, so just ignore it if was
# somehow deleted between entry of this method
# and here
if os.path.exists(volume_path):
os.unlink(volume_path)
else:
LOG.debug(
'Volume path %s not found at end, '
'of remove_iscsi_target.', volume_path)
def _convert_image(prefix,
source,
dest,
out_format,
src_format=None,
run_as_root=True):
"""Convert image to other format."""
# WRS: Append '-W' option to write out-of-order instead of sequential.
cmd = prefix + ('qemu-img', 'convert', '-W', '-O', out_format, source,
dest)
# Check whether O_DIRECT is supported and set '-t none' if it is
# This is needed to ensure that all data hit the device before
# it gets unmapped remotely from the host for some backends
# Reference Bug: #1363016
# NOTE(jdg): In the case of file devices qemu does the
# flush properly and more efficiently than would be done
# setting O_DIRECT, so check for that and skip the
# setting for non BLK devs
if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(
source, dest, 'oflag=direct')):
cmd = prefix + ('qemu-img', 'convert', '-t', 'none', '-W')
# AMI images can be raw or qcow2 but qemu-img doesn't accept "ami" as
# an image format, so we use automatic detection.
# TODO(geguileo): This fixes unencrypted AMI image case, but we need to
# fix the encrypted case.
if (src_format or '').lower() not in ('', 'ami'):
cmd += ('-f', src_format) # prevent detection of format
cmd += ('-O', out_format, source, dest)
start_time = timeutils.utcnow()
# If there is not enough space on the conversion partition, include
# the partitions's name in the error message.
try:
utils.execute(*cmd, run_as_root=run_as_root)
except processutils.ProcessExecutionError as ex:
if "No space left" in ex.stderr and CONF.image_conversion_dir in dest:
conversion_dir = CONF.image_conversion_dir
while not os.path.ismount(conversion_dir):
conversion_dir = os.path.dirname(conversion_dir)
raise exception.InsufficientConversionSpace(
location=conversion_dir)
raise
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
# NOTE(jdg): use a default of 1, mostly for unit test, but in
# some incredible event this is 0 (cirros image?) don't barf
if duration < 1:
duration = 1
try:
image_size = qemu_img_info(source,
run_as_root=run_as_root).virtual_size
except ValueError as e:
msg = ("The image was successfully converted, but image size "
"is unavailable. src %(src)s, dest %(dest)s. %(error)s")
LOG.info(msg, {"src": source, "dest": dest, "error": e})
return
fsz_mb = image_size / units.Mi
mbps = (fsz_mb / duration)
msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
"duration %(duration).2f sec, destination %(dest)s")
LOG.debug(msg, {
"src": source,
"sz": fsz_mb,
"duration": duration,
"dest": dest
})
msg = ("Converted %(sz).2f MB image at %(mbps).2f MB/s, "
"duration %(dur).1f sec")
LOG.info(msg, {"sz": fsz_mb, "mbps": mbps, 'dur': duration})
def create_iscsi_target(self,
name,
tid,
lun,
path,
chap_auth=None,
**kwargs):
# Note(jdg) tid and lun aren't used by TgtAdm but remain for
# compatibility
# NOTE(jdg): Remove this when we get to the bottom of bug: #1398078
# for now, since we intermittently hit target already exists we're
# adding some debug info to try and pinpoint what's going on
(out, err) = utils.execute('tgtadm',
'--lld',
'iscsi',
'--op',
'show',
'--mode',
'target',
run_as_root=True)
LOG.debug("Targets prior to update: %s", out)
fileutils.ensure_tree(self.volumes_dir)
vol_id = name.split(':')[1]
write_cache = self.configuration.get('iscsi_write_cache', 'on')
driver = self.iscsi_protocol
chap_str = ''
if chap_auth is not None:
chap_str = 'incominguser %s %s' % chap_auth
target_flags = self.configuration.get('iscsi_target_flags', '')
if target_flags:
target_flags = 'bsoflags ' + target_flags
volume_conf = self.VOLUME_CONF % {
'name': name,
'path': path,
'driver': driver,
'chap_auth': chap_str,
'target_flags': target_flags,
'write_cache': write_cache
}
LOG.debug('Creating iscsi_target for Volume ID: %s', vol_id)
volumes_dir = self.volumes_dir
volume_path = os.path.join(volumes_dir, vol_id)
if os.path.exists(volume_path):
LOG.warning(
_LW('Persistence file already exists for volume, '
'found file at: %s'), volume_path)
f = open(volume_path, 'w+')
f.write(volume_conf)
f.close()
LOG.debug(('Created volume path %(vp)s,\n'
'content: %(vc)s'), {
'vp': volume_path,
'vc': volume_conf
})
old_persist_file = None
old_name = kwargs.get('old_name', None)
if old_name is not None:
LOG.debug(
'Detected old persistence file for volume '
'%{vol}s at %{old_name}s', {
'vol': vol_id,
'old_name': old_name
})
old_persist_file = os.path.join(volumes_dir, old_name)
try:
# With the persistent tgts we create them
# by creating the entry in the persist file
# and then doing an update to get the target
# created.
self._do_tgt_update(name)
except putils.ProcessExecutionError as e:
if "target already exists" in e.stderr:
# Adding the additional Warning message below for a clear
# ER marker (Ref bug: #1398078).
LOG.warning(
_LW('Could not create target because '
'it already exists for volume: %s'), vol_id)
LOG.debug('Exception was: %s', e)
else:
LOG.error(
_LE("Failed to create iscsi target for Volume "
"ID: %(vol_id)s: %(e)s"), {
'vol_id': vol_id,
'e': e
})
# Don't forget to remove the persistent file we created
os.unlink(volume_path)
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
# Grab targets list for debug
# Consider adding a check for lun 0 and 1 for tgtadm
# before considering this as valid
(out, err) = utils.execute('tgtadm',
'--lld',
'iscsi',
'--op',
'show',
'--mode',
'target',
run_as_root=True)
LOG.debug("Targets after update: %s", out)
iqn = '%s%s' % (self.iscsi_target_prefix, vol_id)
tid = self._get_target(iqn)
if tid is None:
LOG.error(
_LE("Failed to create iscsi target for Volume "
"ID: %(vol_id)s. Please ensure your tgtd config "
"file contains 'include %(volumes_dir)s/*'"), {
'vol_id': vol_id,
'volumes_dir': volumes_dir,
})
raise exception.NotFound()
# NOTE(jdg): Sometimes we have some issues with the backing lun
# not being created, believe this is due to a device busy
# or something related, so we're going to add some code
# here that verifies the backing lun (lun 1) was created
# and we'll try and recreate it if it's not there
if not self._verify_backing_lun(iqn, tid):
try:
self._recreate_backing_lun(iqn, tid, name, path)
except putils.ProcessExecutionError:
os.unlink(volume_path)
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
# Finally check once more and if no go, fail and punt
if not self._verify_backing_lun(iqn, tid):
os.unlink(volume_path)
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
if old_persist_file is not None and os.path.exists(old_persist_file):
os.unlink(old_persist_file)
return tid
def delete(self, backup):
"""Delete the given backup from TSM server.
:param backup: backup information for volume
:raises InvalidBackup
"""
delete_attrs = {'Total number of objects deleted': '1'}
volume_id = backup['volume_id']
backup_id = backup['id']
LOG.debug('delete started, backup: %s', backup['id'])
volume_path = utils.make_dev_path('%s-%s' %
(self.volume_prefix, backup_id))
try:
out, err = utils.execute('dsmc',
'delete',
'backup',
'-quiet',
'-noprompt',
'-objtype=image',
'-deltype=all',
'-password=%s' % self.tsm_password,
volume_path,
run_as_root=True,
check_exit_code=False)
except processutils.ProcessExecutionError as e:
err = (_('delete: %(vol_id)s Failed to run dsmc with '
'stdout: %(out)s\n stderr: %(err)s') % {
'vol_id': volume_id,
'out': e.stdout,
'err': e.stderr
})
LOG.error(err)
raise exception.InvalidBackup(reason=err)
except exception.Error as e:
err = (_('restore: %(vol_id)s Failed to run dsmc '
'due to invalid arguments with '
'stdout: %(out)s\n stderr: %(err)s') % {
'vol_id': volume_id,
'out': e.stdout,
'err': e.stderr
})
LOG.error(err)
raise exception.InvalidBackup(reason=err)
success = self._check_dsmc_output(out, delete_attrs)
if not success:
err = (_('delete: %(vol_id)s Failed with '
'stdout: %(out)s\n stderr: %(err)s') % {
'vol_id': volume_id,
'out': out,
'err': err
})
LOG.error(err)
raise exception.InvalidBackup(reason=err)
LOG.debug(_('delete %s finished') % backup['id'])
def _execute_wrapper(self, cmd, *args, **kwargs):
try:
kwargs.pop('run_as_root')
except KeyError:
pass
return utils.execute(cmd, *args, **kwargs)
def convert_image(source, dest, out_format):
"""Convert image to other format"""
cmd = ('qemu-img', 'convert', '-O', out_format, source, dest)
utils.execute(*cmd, run_as_root=True)
def _convert_image(prefix,
source,
dest,
out_format,
src_format=None,
run_as_root=True):
"""Convert image to other format."""
cmd = prefix + ('qemu-img', 'convert', '-O', out_format, source, dest)
# Check whether O_DIRECT is supported and set '-t none' if it is
# This is needed to ensure that all data hit the device before
# it gets unmapped remotely from the host for some backends
# Reference Bug: #1363016
# NOTE(jdg): In the case of file devices qemu does the
# flush properly and more efficiently than would be done
# setting O_DIRECT, so check for that and skip the
# setting for non BLK devs
if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(
source, dest, 'oflag=direct')):
cmd = prefix + ('qemu-img', 'convert', '-t', 'none')
# AMI images can be raw or qcow2 but qemu-img doesn't accept "ami" as
# an image format, so we use automatic detection.
# TODO(geguileo): This fixes unencrypted AMI image case, but we need to
# fix the encrypted case.
if (src_format or '').lower() not in ('', 'ami'):
cmd += ('-f', src_format) # prevent detection of format
cmd += ('-O', out_format, source, dest)
start_time = timeutils.utcnow()
utils.execute(*cmd, run_as_root=run_as_root)
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
# NOTE(jdg): use a default of 1, mostly for unit test, but in
# some incredible event this is 0 (cirros image?) don't barf
if duration < 1:
duration = 1
try:
image_size = qemu_img_info(source,
run_as_root=run_as_root).virtual_size
except ValueError as e:
msg = ("The image was successfully converted, but image size "
"is unavailable. src %(src)s, dest %(dest)s. %(error)s")
LOG.info(msg, {"src": source, "dest": dest, "error": e})
return
fsz_mb = image_size / units.Mi
mbps = (fsz_mb / duration)
msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
"duration %(duration).2f sec, destination %(dest)s")
LOG.debug(msg, {
"src": source,
"sz": fsz_mb,
"duration": duration,
"dest": dest
})
msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s"
LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def _fake_convert_image(self, source, dest, out_format):
utils.execute('cp', source, dest)
def coalesce_vhd(vhd_path: str) -> None:
utils.execute('vhd-util', 'coalesce', '-n', vhd_path)
def _fake_create_file(self, path, modebits='666'):
open(path, 'w').close()
utils.execute('chmod', modebits, path)
def _remove_fake_mount(self):
utils.execute('rm', '-rf', self.TEST_MOUNT)
def _change_file_mode(filepath):
utils.execute('chmod', '640', filepath, run_as_root=True)
def get_vhd_size(vhd_path):
out, err = utils.execute('vhd-util', 'query', '-n', vhd_path, '-v')
return int(out)
def resize_image(source, size):
"""Changes the virtual size of the image."""
cmd = ('qemu-img', 'resize', source, '%sG' % size)
utils.execute(*cmd, run_as_root=False)
def extract_targz(archive_name, target):
utils.execute('tar', '-xzf', archive_name, '-C', target)
def delete_backup(self, backup):
"""Delete the given backup from TSM server.
:param backup: backup information for volume
:raises: InvalidBackup
"""
delete_attrs = {'Total number of objects deleted': '1'}
delete_path, backup_mode = _get_backup_metadata(backup, 'restore')
LOG.debug('Delete started for backup: %(backup)s, mode: %(mode)s.', {
'backup': backup.id,
'mode': backup_mode
})
try:
out, err = utils.execute('dsmc',
'delete',
'backup',
'-quiet',
'-noprompt',
'-objtype=%s' % backup_mode,
'-password=%s' % self.tsm_password,
delete_path,
run_as_root=True,
check_exit_code=False)
except processutils.ProcessExecutionError as exc:
err = (_('delete: %(vol_id)s failed to run dsmc with '
'stdout: %(out)s\n stderr: %(err)s') % {
'vol_id': backup.volume_id,
'out': exc.stdout,
'err': exc.stderr
})
LOG.error(err)
raise exception.InvalidBackup(reason=err)
except exception.Error as exc:
err = (_('delete: %(vol_id)s failed to run dsmc '
'due to invalid arguments with '
'stdout: %(out)s\n stderr: %(err)s') % {
'vol_id': backup.volume_id,
'out': exc.stdout,
'err': exc.stderr
})
LOG.error(err)
raise exception.InvalidBackup(reason=err)
success = _check_dsmc_output(out, delete_attrs)
if not success:
# log error if tsm cannot delete the backup object
# but do not raise exception so that cinder backup
# object can be removed.
LOG.error(
'delete: %(vol_id)s failed with '
'stdout: %(out)s\n stderr: %(err)s', {
'vol_id': backup.volume_id,
'out': out,
'err': err
})
LOG.debug('Delete %s finished.', backup['id'])
