def convert_image(source, dest, out_format, bps_limit=None): """Convert image to other format.""" start_time = timeutils.utcnow() # Always set -t none. First it is needed for cgroup io/limiting # and it is needed to ensure that all data hit the device before # it gets unmapped remotely from the host cmd = ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % { "src": source, "sz": fsz_mb, "duration": duration, "dest": dest }) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def convert_image(source, dest, out_format, bps_limit=None): """Convert image to other format.""" start_time = timeutils.utcnow() # Always set -t none. First it is needed for cgroup io/limiting # and it is needed to ensure that all data hit the device before # it gets unmapped remotely from the host cmd = ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def fetch(context, image_service, image_id, path, _user_id, _project_id): # TODO(vish): Improve context handling and add owner and auth data # when it is added to glance. Right now there is no # auth checking in glance, so we assume that access was # checked before we got here. start_time = timeutils.utcnow() with fileutils.remove_path_on_error(path): with open(path, "wb") as image_file: image_service.download(context, image_id, image_file) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(image_file.name).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image fetch details: dest %(dest)s, size %(sz).2f MB, " "duration %(duration).2f sec") LOG.debug(msg % { "dest": image_file.name, "sz": fsz_mb, "duration": duration }) msg = _("Image download %(sz).2f MB at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" idle_for = DEFAULT_INTERVAL for task_name, task in self._periodic_tasks: full_task_name = '.'.join([self.__class__.__name__, task_name]) now = timeutils.utcnow() spacing = self._periodic_spacing[task_name] last_run = self._periodic_last_run[task_name] # If a periodic task is _nearly_ due, then we'll run it early if spacing is not None and last_run is not None: due = last_run + datetime.timedelta(seconds=spacing) if not timeutils.is_soon(due, 0.2): idle_for = min(idle_for, timeutils.delta_seconds(now, due)) continue if spacing is not None: idle_for = min(idle_for, spacing) LOG.debug(_("Running periodic task %(full_task_name)s"), locals()) self._periodic_last_run[task_name] = timeutils.utcnow() try: task(self, context) except Exception as e: if raise_on_error: raise LOG.exception(_("Error during %(full_task_name)s: %(e)s"), locals()) time.sleep(0) return idle_for
def convert_image(source, dest, out_format, bps_limit=None): """Convert image to other format.""" start_time = timeutils.utcnow() cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd cmd += ('-t', 'none') # required to enable ratelimit by blkio cgroup utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def list_availability_zones(self, enable_cache=False): """Describe the known availability zones :retval list of dicts, each with a 'name' and 'available' key """ refresh_cache = False if enable_cache: if self.availability_zones_last_fetched is None: refresh_cache = True else: cache_age = timeutils.delta_seconds(self.availability_zones_last_fetched, timeutils.utcnow()) if cache_age >= CONF.az_cache_duration: refresh_cache = True if refresh_cache or not enable_cache: topic = CONF.volume_topic ctxt = context.get_admin_context() services = self.db.service_get_all_by_topic(ctxt, topic) az_data = [(s["availability_zone"], s["disabled"]) for s in services] disabled_map = {} for (az_name, disabled) in az_data: tracked_disabled = disabled_map.get(az_name, True) disabled_map[az_name] = tracked_disabled and disabled azs = [{"name": name, "available": not disabled} for (name, disabled) in disabled_map.items()] if refresh_cache: now = timeutils.utcnow() self.availability_zones = azs self.availability_zones_last_fetched = now LOG.debug( "Availability zone cache updated, next update will" " occur around %s", now + datetime.timedelta(seconds=CONF.az_cache_duration), ) else: azs = self.availability_zones return tuple(azs)
def convert_image(source, dest, out_format, bps_limit=None): """Convert image to other format.""" start_time = timeutils.utcnow() cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd cmd += ('-t', 'none') # required to enable ratelimit by blkio cgroup utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % { "src": source, "sz": fsz_mb, "duration": duration, "dest": dest }) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def copy_volume(srcstr, deststr, size_in_m, blocksize, sync=False, execute=utils.execute, ionice=None): # Use O_DIRECT to avoid thrashing the system buffer cache extra_flags = [] if check_for_odirect_support(srcstr, deststr, 'iflag=direct'): extra_flags.append('iflag=direct') if check_for_odirect_support(srcstr, deststr, 'oflag=direct'): extra_flags.append('oflag=direct') # If the volume is being unprovisioned then # request the data is persisted before returning, # so that it's not discarded from the cache. if sync and not extra_flags: extra_flags.append('conv=fdatasync') blocksize, count = _calculate_count(size_in_m, blocksize) cmd = [ 'dd', 'if=%s' % srcstr, 'of=%s' % deststr, 'count=%d' % count, 'bs=%s' % blocksize ] cmd.extend(extra_flags) if ionice is not None: cmd = ['ionice', ionice] + cmd cgcmd = setup_blkio_cgroup(srcstr, deststr, CONF.volume_copy_bps_limit) if cgcmd: cmd = cgcmd + cmd # Perform the copy start_time = timeutils.utcnow() execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 mbps = (size_in_m / duration) mesg = ("Volume copy details: src %(src)s, dest %(dest)s, " "size %(sz).2f MB, duration %(duration).2f sec") LOG.debug(mesg % { "src": srcstr, "dest": deststr, "sz": size_in_m, "duration": duration }) mesg = _("Volume copy %(size_in_m).2f MB at %(mbps).2f MB/s") LOG.info(mesg % {'size_in_m': size_in_m, 'mbps': mbps})
def convert_image(source, dest, out_format, bps_limit=None, is_qcow_compress=False): """Convert image to other format.""" cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) if is_qcow_compress and out_format=='qcow2': cmd = ('qemu-img', 'convert', '-c', '-O', out_format, source, dest) else: cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): cmd = ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) start_time = timeutils.utcnow() cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % {"src": source, "sz": fsz_mb, "duration": duration, "dest": dest}) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def convert_image(source, dest, out_format, bps_limit=None, is_qcow_compress=False): """Convert image to other format.""" cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) if is_qcow_compress and out_format == 'qcow2': cmd = ('qemu-img', 'convert', '-c', '-O', out_format, source, dest) else: cmd = ('qemu-img', 'convert', '-O', out_format, source, dest) # Check whether O_DIRECT is supported and set '-t none' if it is # This is needed to ensure that all data hit the device before # it gets unmapped remotely from the host for some backends # Reference Bug: #1363016 # NOTE(jdg): In the case of file devices qemu does the # flush properly and more efficiently than would be done # setting O_DIRECT, so check for that and skip the # setting for non BLK devs if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support( source, dest, 'oflag=direct')): cmd = ('qemu-img', 'convert', '-t', 'none', '-O', out_format, source, dest) start_time = timeutils.utcnow() cgcmd = volume_utils.setup_blkio_cgroup(source, dest, bps_limit) if cgcmd: cmd = tuple(cgcmd) + cmd utils.execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(source).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, " "duration %(duration).2f sec, destination %(dest)s") LOG.debug(msg % { "src": source, "sz": fsz_mb, "duration": duration, "dest": dest }) msg = _("Converted %(sz).2f MB image at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def copy_volume(srcstr, deststr, size_in_m, blocksize, sync=False, execute=utils.execute, ionice=None): # Use O_DIRECT to avoid thrashing the system buffer cache extra_flags = [] # Check whether O_DIRECT is supported to iflag and oflag separately for flag in ['iflag=direct', 'oflag=direct']: try: execute('dd', 'count=0', 'if=%s' % srcstr, 'of=%s' % deststr, flag, run_as_root=True) extra_flags.append(flag) except processutils.ProcessExecutionError: pass # If the volume is being unprovisioned then # request the data is persisted before returning, # so that it's not discarded from the cache. if sync and not extra_flags: extra_flags.append('conv=fdatasync') blocksize, count = _calculate_count(size_in_m, blocksize) cmd = ['dd', 'if=%s' % srcstr, 'of=%s' % deststr, 'count=%d' % count, 'bs=%s' % blocksize] cmd.extend(extra_flags) if ionice is not None: cmd = ['ionice', ionice] + cmd cgcmd = setup_blkio_cgroup(srcstr, deststr, CONF.volume_copy_bps_limit) if cgcmd: cmd = cgcmd + cmd # Perform the copy start_time = timeutils.utcnow() execute(*cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 mbps = (size_in_m / duration) mesg = ("Volume copy details: src %(src)s, dest %(dest)s, " "size %(sz).2f MB, duration %(duration).2f sec") LOG.debug(mesg % {"src": srcstr, "dest": deststr, "sz": size_in_m, "duration": duration}) mesg = _("Volume copy %(size_in_m).2f MB at %(mbps).2f MB/s") LOG.info(mesg % {'size_in_m': size_in_m, 'mbps': mbps})
def clear_volume(volume_size, volume_path, volume_clear=None, volume_clear_size=None, volume_clear_ionice=None): """Unprovision old volumes to prevent data leaking between users.""" if volume_clear is None: volume_clear = CONF.volume_clear if volume_clear_size is None: volume_clear_size = CONF.volume_clear_size if volume_clear_size == 0: volume_clear_size = volume_size if volume_clear_ionice is None: volume_clear_ionice = CONF.volume_clear_ionice LOG.info(_("Performing secure delete on volume: %s") % volume_path) if volume_clear == 'zero': return copy_volume('/dev/zero', volume_path, volume_clear_size, CONF.volume_dd_blocksize, sync=True, execute=utils.execute, ionice=volume_clear_ionice) elif volume_clear == 'shred': clear_cmd = ['shred', '-n3'] if volume_clear_size: clear_cmd.append('-s%dMiB' % volume_clear_size) else: raise exception.InvalidConfigurationValue(option='volume_clear', value=volume_clear) clear_cmd.append(volume_path) start_time = timeutils.utcnow() utils.execute(*clear_cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 LOG.info(_('Elapsed time for clear volume: %.2f sec') % duration)
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone, e: self.stop() done.send(e.retvalue)
def clear_volume(volume_size, volume_path, volume_clear=None, volume_clear_size=None, volume_clear_ionice=None): """Unprovision old volumes to prevent data leaking between users.""" if volume_clear is None: volume_clear = CONF.volume_clear if volume_clear_size is None: volume_clear_size = CONF.volume_clear_size if volume_clear_size == 0: volume_clear_size = volume_size if volume_clear_ionice is None: volume_clear_ionice = CONF.volume_clear_ionice LOG.info(_("Performing secure delete on volume: %s") % volume_path) if volume_clear == 'zero': return copy_volume('/dev/zero', volume_path, volume_clear_size, CONF.volume_dd_blocksize, sync=True, execute=utils.execute, ionice=volume_clear_ionice) elif volume_clear == 'shred': clear_cmd = ['shred', '-n3'] if volume_clear_size: clear_cmd.append('-s%dMiB' % volume_clear_size) else: raise exception.InvalidConfigurationValue( option='volume_clear', value=volume_clear) clear_cmd.append(volume_path) start_time = timeutils.utcnow() utils.execute(*clear_cmd, run_as_root=True) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 LOG.info(_('Elapsed time for clear volume: %.2f sec') % duration)
def fetch(context, image_service, image_id, path, _user_id, _project_id): # TODO(vish): Improve context handling and add owner and auth data # when it is added to glance. Right now there is no # auth checking in glance, so we assume that access was # checked before we got here. start_time = timeutils.utcnow() with fileutils.remove_path_on_error(path): with open(path, "wb") as image_file: image_service.download(context, image_id, image_file) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(image_file.name).st_size / units.Mi mbps = fsz_mb / duration msg = "Image fetch details: dest %(dest)s, size %(sz).2f MB, " "duration %(duration).2f sec" LOG.debug(msg % {"dest": image_file.name, "sz": fsz_mb, "duration": duration}) msg = _("Image download %(sz).2f MB at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def list_availability_zones(self, enable_cache=False): """Describe the known availability zones :retval list of dicts, each with a 'name' and 'available' key """ refresh_cache = False if enable_cache: if self.availability_zones_last_fetched is None: refresh_cache = True else: cache_age = timeutils.delta_seconds( self.availability_zones_last_fetched, timeutils.utcnow()) if cache_age >= CONF.az_cache_duration: refresh_cache = True if refresh_cache or not enable_cache: topic = CONF.volume_topic ctxt = context.get_admin_context() services = self.db.service_get_all_by_topic(ctxt, topic) az_data = [(s['availability_zone'], s['disabled']) for s in services] disabled_map = {} for (az_name, disabled) in az_data: tracked_disabled = disabled_map.get(az_name, True) disabled_map[az_name] = tracked_disabled and disabled azs = [{ 'name': name, 'available': not disabled } for (name, disabled) in disabled_map.items()] if refresh_cache: now = timeutils.utcnow() self.availability_zones = azs self.availability_zones_last_fetched = now LOG.debug( "Availability zone cache updated, next update will" " occur around %s", now + datetime.timedelta(seconds=CONF.az_cache_duration)) else: azs = self.availability_zones return tuple(azs)
def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" idle_for = DEFAULT_INTERVAL for task_name, task in self._periodic_tasks: full_task_name = '.'.join([self.__class__.__name__, task_name]) now = timeutils.utcnow() spacing = self._periodic_spacing[task_name] last_run = self._periodic_last_run[task_name] # If a periodic task is _nearly_ due, then we'll run it early if spacing is not None and last_run is not None: due = last_run + datetime.timedelta(seconds=spacing) if not timeutils.is_soon(due, 0.2): idle_for = min(idle_for, timeutils.delta_seconds(now, due)) continue if spacing is not None: idle_for = min(idle_for, spacing) LOG.debug(_("Running periodic task %(full_task_name)s"), {"full_task_name": full_task_name}) self._periodic_last_run[task_name] = timeutils.utcnow() try: task(self, context) except Exception as e: if raise_on_error: raise LOG.exception(_("Error during %(full_task_name)s: %(e)s"), { "full_task_name": full_task_name, "e": e }) time.sleep(0) return idle_for