def _copy_volume(self,src_vol_dir,des_vol_dir,src_vol_size,vol_dd_size,sync=False): if src_vol_dir is None or des_vol_dir is None: return False bksize,count=vutils._calculate_count(src_vol_size, vol_dd_size) extra_flags = [] if vutils.check_for_odirect_support(src_vol_dir, des_vol_dir, 'iflag=direct'): extra_flags.append('iflag=direct') if vutils.check_for_odirect_support(src_vol_dir, des_vol_dir, 'oflag=direct'): extra_flags.append('oflag=direct') if sync and not extra_flags: extra_flags.append('conv=sync') cmdstr = ['dd', 'if=%s' % src_vol_dir, 'of=%s' % des_vol_dir, 'count=%d' % count, 'bs=%s' % bksize] cmdstr.extend(extra_flags) try: self._execute(*cmdstr,root_helper=self.r_helper,run_as_root=True) except putils.ProcessExecutionError as err: LOG.error(_('Cmd :%s') % err.cmd) LOG.error(_('StdOut :%s') % err.stdout) LOG.error(_('StdErr :%s') % err.stderr) raise NameError('Error:failed to dd data from snapshot:%s' % src_vol_dir) return True
def test_check_for_odirect_support(self, mock_exec): output = volume_utils.check_for_odirect_support('/dev/abc', '/dev/def') self.assertTrue(output) mock_exec.assert_called_once_with('dd', 'count=0', 'if=/dev/abc', 'of=/dev/def', 'oflag=direct', run_as_root=True) mock_exec.reset_mock() output = volume_utils.check_for_odirect_support('/dev/abc', '/dev/def', 'iflag=direct') self.assertTrue(output) mock_exec.assert_called_once_with('dd', 'count=0', 'if=/dev/abc', 'of=/dev/def', 'iflag=direct', run_as_root=True) mock_exec.reset_mock() output = volume_utils.check_for_odirect_support('/dev/zero', '/dev/def', 'iflag=direct') self.assertFalse(output) mock_exec.reset_mock() output = volume_utils.check_for_odirect_support('/dev/zero', '/dev/def') self.assertTrue(output) mock_exec.assert_called_once_with('dd', 'count=0', 'if=/dev/zero', 'of=/dev/def', 'oflag=direct', run_as_root=True)
def test_check_for_odirect_support_error(self, mock_exec): output = volume_utils.check_for_odirect_support("/dev/abc", "/dev/def") self.assertFalse(output) mock_exec.assert_called_once_with( "dd", "count=0", "if=/dev/abc", "of=/dev/def", "oflag=direct", run_as_root=True ) mock_exec.reset_mock() output = volume_utils.check_for_odirect_support("/dev/zero", "/dev/def") self.assertFalse(output) mock_exec.assert_called_once_with( "dd", "count=0", "if=/dev/zero", "of=/dev/def", "oflag=direct", run_as_root=True )
def copy_file_to_volume(self, context, src_file, volume): """Copies a file to a volume.""" if self._volume_not_present(volume['name']): # The underlying volume is gone. We need to re-create it. self.create_volume(volume) elif self.vg.lv_has_snapshot(volume['name']): LOG.error( 'Unable to copy due to existing snapshot ' 'for volume: %s', volume['name']) raise exception.VolumeIsBusy(volume_name=volume['name']) # Use O_DIRECT to avoid thrashing the system buffer cache extra_flags = [] if volutils.check_for_odirect_support(src_file, self.local_path(volume), 'iflag=direct'): extra_flags.append('iflag=direct') if volutils.check_for_odirect_support(src_file, self.local_path(volume), 'oflag=direct'): extra_flags.append('oflag=direct') conv = [] if not extra_flags: conv.append('fdatasync') if conv: conv_options = 'conv=' + ",".join(conv) extra_flags.append(conv_options) try: size_in_bytes = int(volume['size']) * 1024**3 # vol size is GB blocksize = volutils._check_blocksize( self.configuration.volume_dd_blocksize) # Perform the copy cmd = [ 'dd', 'if=%s' % src_file, 'of=%s' % self.local_path(volume), 'count=%d' % size_in_bytes, 'bs=%s' % blocksize ] cmd.extend(extra_flags) utils.execute(*cmd, run_as_root=True) except Exception: msg = (_("Failed to copy %(src)s to volume %(dest)s") % { 'src': src_file, 'dest': volume['id'] }) LOG.error(msg) raise self.restore_configuration()
def _convert_image(prefix, source, dest, out_format, out_subformat=None, src_format=None, run_as_root=True): """Convert image to other format.""" # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): cache_mode = 'none' else: # use default cache_mode = None cmd = _get_qemu_convert_cmd(source, dest, out_format=out_format, src_format=src_format, out_subformat=out_subformat, cache_mode=cache_mode, prefix=prefix) start_time = timeutils.utcnow() utils.execute(*cmd, run_as_root=run_as_root) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 try: image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size except ValueError as e: msg = ("The image was successfully converted, but image size " "is unavailable. src %(src)s, dest %(dest)s. %(error)s") LOG.info(msg, {"src": source, "dest": dest, "error": e}) return fsz_mb = image_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg, {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s" LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def copy_snapshot_to_file(self, snapshot, dest_file): """Copies a snapshot to a file.""" # Some configurations of LVM do not automatically activate # ThinLVM snapshot LVs. self.vg.activate_lv(snapshot['name'], is_snapshot=True) # Use O_DIRECT to avoid thrashing the system buffer cache extra_flags = [] if volutils.check_for_odirect_support(self.local_path(snapshot), dest_file, 'iflag=direct'): extra_flags.append('iflag=direct') if volutils.check_for_odirect_support(self.local_path(snapshot), dest_file, 'oflag=direct'): extra_flags.append('oflag=direct') conv = [] if not extra_flags: conv.append('fdatasync') if conv: conv_options = 'conv=' + ",".join(conv) extra_flags.append(conv_options) try: size_in_bytes = int(snapshot['volume_size']) * 1024**3 blocksize = volutils._check_blocksize( self.configuration.volume_dd_blocksize) # Perform the copy cmd = [ 'dd', 'if=%s' % self.local_path(snapshot), 'of=%s' % dest_file, 'count=%d' % size_in_bytes, 'bs=%s' % blocksize ] cmd.extend(extra_flags) utils.execute(*cmd, run_as_root=True) except Exception: msg = (_("Failed to export snapshot %(src)s to %(dest)s") % { 'src': snapshot['id'], 'dest': dest_file }) LOG.error(msg) raise
def convert_image(source, dest, out_format, bps_limit=None, is_qcow_compress=False): """Convert image to other format.""" cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) if is_qcow_compress and out_format=='qcow2': cmd = ('qemu-img', 'convert', '-c', '-O', out_format, source, dest) else: cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): cmd = ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) start_time = timeutils.utcnow() cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def convert_image(source, dest, out_format, bps_limit=None, is_qcow_compress=False): """Convert image to other format.""" cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) if is_qcow_compress and out_format == 'qcow2': cmd = ('qemu-img', 'convert', '-c', '-O', out_format, source, dest) else: cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support( source, dest, 'oflag=direct')): cmd = ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) start_time = timeutils.utcnow() cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % { "src": source, "sz": fsz_mb, "duration": duration, "dest": dest }) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def copy_volume_to_file(self, context, volume, dest_file): """Copies a volume to a file.""" # Use O_DIRECT to avoid thrashing the system buffer cache extra_flags = [] if volutils.check_for_odirect_support(self.local_path(volume), dest_file, 'iflag=direct'): extra_flags.append('iflag=direct') if volutils.check_for_odirect_support(self.local_path(volume), dest_file, 'oflag=direct'): extra_flags.append('oflag=direct') conv = [] if not extra_flags: conv.append('fdatasync') if conv: conv_options = 'conv=' + ",".join(conv) extra_flags.append(conv_options) try: size_in_bytes = int(volume['size']) * 1024**3 # vol size is GB blocksize = volutils._check_blocksize( self.configuration.volume_dd_blocksize) # Perform the copy cmd = [ 'dd', 'if=%s' % self.local_path(volume), 'of=%s' % dest_file, 'count=%d' % size_in_bytes, 'bs=%s' % blocksize, 'status=none' ] cmd.extend(extra_flags) self._execute(*cmd, run_as_root=True) except Exception: msg = (_("Failed to copy volume %(src)s to %(dest)s") % { 'src': volume['id'], 'dest': dest_file }) LOG.error(msg) raise
def _convert_image(prefix, source, dest, out_format, run_as_root=True): """Convert image to other format.""" cmd = prefix + ('qemu-img', 'convert', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): cmd = prefix + ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) start_time = timeutils.utcnow() utils.execute(*cmd, run_as_root=run_as_root) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 try: image_size = qemu_img_info(source, run_as_root=True).virtual_size except ValueError as e: msg = _LI("The image was successfully converted, but image size " "is unavailable. src %(src)s, dest %(dest)s. %(error)s") LOG.info(msg, {"src": source, "dest": dest, "error": e}) return fsz_mb = image_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg, {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = _LI("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def _convert_image(prefix, source, dest, out_format, src_format=None, run_as_root=True): """Convert image to other format.""" cmd = prefix + ('qemu-img', 'convert', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): cmd = prefix + ('qemu-img', 'convert', '-t', 'none') # AMI images can be raw or qcow2 but qemu-img doesn't accept "ami" as # an image format, so we use automatic detection. # TODO(geguileo): This fixes unencrypted AMI image case, but we need to # fix the encrypted case. if (src_format or '').lower() not in ('', 'ami'): cmd += ('-f', src_format) # prevent detection of format cmd += ('-O', out_format, source, dest) start_time = timeutils.utcnow() utils.execute(*cmd, run_as_root=run_as_root) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 try: image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size except ValueError as e: msg = ("The image was successfully converted, but image size " "is unavailable. src %(src)s, dest %(dest)s. %(error)s") LOG.info(msg, {"src": source, "dest": dest, "error": e}) return fsz_mb = image_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg, {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s" LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def _convert_image(prefix, source, dest, out_format, out_subformat=None, src_format=None, run_as_root=True, cipher_spec=None, passphrase_file=None): """Convert image to other format.""" # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support( source, dest, 'oflag=direct')): cache_mode = 'none' else: # use default cache_mode = None cmd = _get_qemu_convert_cmd(source, dest, out_format=out_format, src_format=src_format, out_subformat=out_subformat, cache_mode=cache_mode, prefix=prefix, cipher_spec=cipher_spec, passphrase_file=passphrase_file) start_time = timeutils.utcnow() # If there is not enough space on the conversion partition, include # the partitions's name in the error message. try: utils.execute(*cmd, run_as_root=run_as_root) except processutils.ProcessExecutionError as ex: if "No space left" in ex.stderr and CONF.image_conversion_dir in dest: conversion_dir = CONF.image_conversion_dir while not os.path.ismount(conversion_dir): conversion_dir = os.path.dirname(conversion_dir) message = _("Insufficient free space on %(location)s for image " "conversion.") % { 'location': conversion_dir } LOG.error(message) raise duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 try: image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size except ValueError as e: msg = ("The image was successfully converted, but image size " "is unavailable. src %(src)s, dest %(dest)s. %(error)s") LOG.info(msg, {"src": source, "dest": dest, "error": e}) return fsz_mb = image_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg, { "src": source, "sz": fsz_mb, "duration": duration, "dest": dest }) msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s" LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
def _convert_image(prefix, source, dest, out_format, src_format=None, run_as_root=True): """Convert image to other format.""" # WRS: Append '-W' option to write out-of-order instead of sequential. cmd = prefix + ('qemu-img', 'convert', '-W', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support( source, dest, 'oflag=direct')): cmd = prefix + ('qemu-img', 'convert', '-t', 'none', '-W') # AMI images can be raw or qcow2 but qemu-img doesn't accept "ami" as # an image format, so we use automatic detection. # TODO(geguileo): This fixes unencrypted AMI image case, but we need to # fix the encrypted case. if (src_format or '').lower() not in ('', 'ami'): cmd += ('-f', src_format) # prevent detection of format cmd += ('-O', out_format, source, dest) start_time = timeutils.utcnow() # If there is not enough space on the conversion partition, include # the partitions's name in the error message. try: utils.execute(*cmd, run_as_root=run_as_root) except processutils.ProcessExecutionError as ex: if "No space left" in ex.stderr and CONF.image_conversion_dir in dest: conversion_dir = CONF.image_conversion_dir while not os.path.ismount(conversion_dir): conversion_dir = os.path.dirname(conversion_dir) raise exception.InsufficientConversionSpace( location=conversion_dir) raise duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 try: image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size except ValueError as e: msg = ("The image was successfully converted, but image size " "is unavailable. src %(src)s, dest %(dest)s. %(error)s") LOG.info(msg, {"src": source, "dest": dest, "error": e}) return fsz_mb = image_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg, { "src": source, "sz": fsz_mb, "duration": duration, "dest": dest }) msg = ("Converted %(sz).2f MB image at %(mbps).2f MB/s, " "duration %(dur).1f sec") LOG.info(msg, {"sz": fsz_mb, "mbps": mbps, 'dur': duration})
def _convert_image(prefix, source, dest, out_format, out_subformat=None, src_format=None, run_as_root=True, cipher_spec=None, passphrase_file=None): """Convert image to other format.""" # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): cache_mode = 'none' else: # use default cache_mode = None cmd = _get_qemu_convert_cmd(source, dest, out_format=out_format, src_format=src_format, out_subformat=out_subformat, cache_mode=cache_mode, prefix=prefix, cipher_spec=cipher_spec, passphrase_file=passphrase_file) start_time = timeutils.utcnow() # If there is not enough space on the conversion partition, include # the partitions's name in the error message. try: utils.execute(*cmd, run_as_root=run_as_root) except processutils.ProcessExecutionError as ex: if "No space left" in ex.stderr and CONF.image_conversion_dir in dest: conversion_dir = CONF.image_conversion_dir while not os.path.ismount(conversion_dir): conversion_dir = os.path.dirname(conversion_dir) message = _("Insufficient free space on %(location)s for image " "conversion.") % {'location': conversion_dir} LOG.error(message) raise duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 try: image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size except ValueError as e: msg = ("The image was successfully converted, but image size " "is unavailable. src %(src)s, dest %(dest)s. %(error)s") LOG.info(msg, {"src": source, "dest": dest, "error": e}) return fsz_mb = image_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg, {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s" LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})