def upload_volume(context, image_service, image_meta, volume_path): image_id = image_meta['id'] if (image_meta['disk_format'] == 'raw'): LOG.debug("%s was raw, no need to convert to %s" % (image_id, image_meta['disk_format'])) with utils.temporary_chown(volume_path): with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was raw, converting to %s" % (image_id, image_meta['disk_format'])) convert_image(volume_path, tmp, image_meta['disk_format']) data = qemu_img_info(tmp) if data.file_format != image_meta['disk_format']: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % { 'f1': image_meta['disk_format'], 'f2': data.file_format }) with fileutils.file_open(tmp) as image_file: image_service.update(context, image_id, {}, image_file) os.unlink(tmp)
def fetch(context, image_service, image_id, path, _user_id, _project_id): # TODO(vish): Improve context handling and add owner and auth data # when it is added to glance. Right now there is no # auth checking in glance, so we assume that access was # checked before we got here. start_time = timeutils.utcnow() with fileutils.remove_path_on_error(path): with open(path, "wb") as image_file: image_service.download(context, image_id, image_file) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(image_file.name).st_size / units.Mi mbps = (fsz_mb / duration) msg = ("Image fetch details: dest %(dest)s, size %(sz).2f MB, " "duration %(duration).2f sec") LOG.debug(msg % { "dest": image_file.name, "sz": fsz_mb, "duration": duration }) msg = _("Image download %(sz).2f MB at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def upload_volume(context, image_service, image_meta, volume_path, volume_format="raw"): image_id = image_meta["id"] if image_meta["disk_format"] == volume_format: LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta["disk_format"])) if os.name == "nt": with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) with utils.temporary_chown(volume_path): with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir): os.makedirs(CONF.image_conversion_dir) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta["disk_format"])) convert_image(volume_path, tmp, image_meta["disk_format"]) data = qemu_img_info(tmp) if data.file_format != image_meta["disk_format"]: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {"f1": image_meta["disk_format"], "f2": data.file_format}, ) with fileutils.file_open(tmp) as image_file: image_service.update(context, image_id, {}, image_file) fileutils.delete_if_exists(tmp)
def fetch_verify_image(context, image_service, image_id, dest, user_id=None, project_id=None, size=None): fetch(context, image_service, image_id, dest, None, None) with fileutils.remove_path_on_error(dest): data = qemu_img_info(dest) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") % { 'fmt': fmt, 'backing_file': backing_file })) # NOTE(xqueralt): If the image virtual size doesn't fit in the # requested volume there is no point on resizing it because it will # generate an unusable image. if size is not None and data.virtual_size > size: params = {'image_size': data.virtual_size, 'volume_size': size} reason = _("Size is %(image_size)dGB and doesn't fit in a " "volume of size %(volume_size)dGB.") % params raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
def fetch_verify_image(context, image_service, image_id, dest, user_id=None, project_id=None, size=None): fetch(context, image_service, image_id, dest, None, None) with fileutils.remove_path_on_error(dest): data = qemu_img_info(dest) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file})) # NOTE(xqueralt): If the image virtual size doesn't fit in the # requested volume there is no point on resizing it because it will # generate an unusable image. if size is not None and data.virtual_size > size: params = {'image_size': data.virtual_size, 'volume_size': size} reason = _("Size is %(image_size)dGB and doesn't fit in a " "volume of size %(volume_size)dGB.") % params raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
def upload_volume(context, image_service, image_meta, volume_path): image_id = image_meta['id'] if (image_meta['disk_format'] == 'raw'): LOG.debug("%s was raw, no need to convert to %s" % (image_id, image_meta['disk_format'])) with utils.temporary_chown(volume_path): with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was raw, converting to %s" % (image_id, image_meta['disk_format'])) convert_image(volume_path, tmp, image_meta['disk_format']) data = qemu_img_info(tmp) if data.file_format != image_meta['disk_format']: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {'f1': image_meta['disk_format'], 'f2': data.file_format}) with fileutils.file_open(tmp) as image_file: image_service.update(context, image_id, {}, image_file) os.unlink(tmp)
def fetch(context, image_service, image_id, path, _user_id, _project_id): # TODO(vish): Improve context handling and add owner and auth data # when it is added to glance. Right now there is no # auth checking in glance, so we assume that access was # checked before we got here. with fileutils.remove_path_on_error(path): with open(path, "wb") as image_file: image_service.download(context, image_id, image_file)
def copy_volume_to_image(self, context, volume, image_service, image_meta): self._ensure_tmp_exists() tmp_dir = self.configuration.volume_tmp_dir or "/tmp" tmp_file = os.path.join(tmp_dir, volume["name"] + "-" + image_meta["id"]) with fileutils.remove_path_on_error(tmp_file): args = ["rbd", "export", "--pool", self.configuration.rbd_pool, volume["name"], tmp_file] args.extend(self._ceph_args()) self._try_execute(*args) image_utils.upload_volume(context, image_service, image_meta, tmp_file) os.unlink(tmp_file)
def copy_volume_to_image(self, context, volume, image_service, image_meta): tmp_dir = self._image_conversion_dir() tmp_file = os.path.join(tmp_dir, volume['name'] + '-' + image_meta['id']) with fileutils.remove_path_on_error(tmp_file): args = ['rbd', 'export', '--pool', self.configuration.rbd_pool, volume['name'], tmp_file] args.extend(self._ceph_args()) self._try_execute(*args) image_utils.upload_volume(context, image_service, image_meta, tmp_file) os.unlink(tmp_file)
def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw'): image_id = image_meta['id'] if (image_meta['disk_format'] == volume_format): LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta['disk_format'])) if os.name == 'nt' or os.access(volume_path, os.R_OK): with fileutils.file_open(volume_path, 'rb') as image_file: image_service.update(context, image_id, {}, image_file) else: with utils.temporary_chown(volume_path): with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta['disk_format'])) data = qemu_img_info(volume_path) backing_file = data.backing_file fmt = data.file_format if backing_file is not None: # Disallow backing files as a security measure. # This prevents a user from writing an image header into a raw # volume with a backing file pointing to data they wish to # access. raise exception.ImageUnacceptable( image_id=image_id, reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file}) convert_image(volume_path, tmp, image_meta['disk_format'], bps_limit=CONF.volume_copy_bps_limit) data = qemu_img_info(tmp) if data.file_format != image_meta['disk_format']: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {'f1': image_meta['disk_format'], 'f2': data.file_format}) with fileutils.file_open(tmp, 'rb') as image_file: image_service.update(context, image_id, {}, image_file) fileutils.delete_if_exists(tmp)
def fetch_to_raw(context, image_service, image_id, dest, user_id=None, project_id=None): if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) # NOTE(avishay): I'm not crazy about creating temp files which may be # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because # it seeks. Maybe we can think of something for a future version. fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): fetch(context, image_service, image_id, tmp, user_id, project_id) data = qemu_img_info(tmp) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=_("fmt=%(fmt)s backed by:" "%(backing_file)s") % { 'fmt': fmt, 'backing_file': backing_file, }) # NOTE(jdg): I'm using qemu-img convert to write # to the volume regardless if it *needs* conversion or not # TODO(avishay): We can speed this up by checking if the image is raw # and if so, writing directly to the device. However, we need to keep # check via 'qemu-img info' that what we copied was in fact a raw # image and not a different format with a backing file, which may be # malicious. LOG.debug("%s was %s, converting to raw" % (image_id, fmt)) convert_image(tmp, dest, 'raw') data = qemu_img_info(dest) if data.file_format != "raw": raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to raw, but format is now %s") % data.file_format) os.unlink(tmp)
def _gpfs_fetch_to_raw(self, context, image_service, image_id, dest, user_id=None, project_id=None): if (self.configuration.image_conversion_dir and not os.path.exists(self.configuration.image_conversion_dir)): os.makedirs(self.configuration.image_conversion_dir) tmp = "%s.part" % dest with fileutils.remove_path_on_error(tmp): self._gpfs_fetch(context, image_service, image_id, tmp, user_id, project_id) data = image_utils.qemu_img_info(tmp) fmt = data.file_format backing_file = data.backing_file if backing_file is not None: msg = (_("fmt = %(fmt)s backed by: %(backing_file)s") % { 'fmt': fmt, 'backing_file': backing_file }) LOG.error(msg) raise exception.ImageUnacceptable(image_id=image_id, reason=msg) if fmt is None: msg = _("'qemu-img info' parsing failed.") LOG.error(msg) raise exception.ImageUnacceptable(reason=msg, image_id=image_id) elif fmt == 'raw': # already in raw format - just rename to dest self._execute('mv', tmp, dest, run_as_root=True) else: # conversion to raw format required LOG.debug("%s was %s, converting to raw" % (image_id, fmt)) image_utils.convert_image(tmp, dest, 'raw') os.unlink(tmp) data = image_utils.qemu_img_info(dest) if data.file_format != "raw": msg = (_("Expected image to be in raw format, but is %s") % data.file_format) LOG.error(msg) raise exception.ImageUnacceptable(image_id=image_id, reason=msg) return {'size': math.ceil(data.virtual_size / 1024.0**3)}
def fetch_verify_image(context, image_service, image_id, dest, user_id=None, project_id=None): fetch(context, image_service, image_id, dest, None, None) with fileutils.remove_path_on_error(dest): data = qemu_img_info(dest) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable(reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") % {"fmt": fmt, "backing_file": backing_file}), )
def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw', run_as_root=True): image_id = image_meta['id'] if (image_meta['disk_format'] == volume_format): LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta['disk_format'])) if os.name == 'nt' or os.access(volume_path, os.R_OK): with fileutils.file_open(volume_path, 'rb') as image_file: image_service.update(context, image_id, {}, image_file) else: with utils.temporary_chown(volume_path): with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta['disk_format'])) convert_image(volume_path, tmp, image_meta['disk_format'], bps_limit=CONF.volume_copy_bps_limit, run_as_root=run_as_root) data = qemu_img_info(tmp, run_as_root=run_as_root) if data.file_format != image_meta['disk_format']: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % { 'f1': image_meta['disk_format'], 'f2': data.file_format }) with fileutils.file_open(tmp, 'rb') as image_file: image_service.update(context, image_id, {}, image_file) fileutils.delete_if_exists(tmp)
def _gpfs_fetch_to_raw(self, context, image_service, image_id, dest, user_id=None, project_id=None): if (self.configuration.image_conversion_dir and not os.path.exists(self.configuration.image_conversion_dir)): os.makedirs(self.configuration.image_conversion_dir) tmp = "%s.part" % dest with fileutils.remove_path_on_error(tmp): self._gpfs_fetch(context, image_service, image_id, tmp, user_id, project_id) data = image_utils.qemu_img_info(tmp) fmt = data.file_format backing_file = data.backing_file if backing_file is not None: msg = (_("fmt = %(fmt)s backed by: %(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file}) LOG.error(msg) raise exception.ImageUnacceptable( image_id=image_id, reason=msg) if fmt is None: msg = _("'qemu-img info' parsing failed.") LOG.error(msg) raise exception.ImageUnacceptable( reason=msg, image_id=image_id) elif fmt == 'raw': # already in raw format - just rename to dest self._execute('mv', tmp, dest, run_as_root=True) else: # conversion to raw format required LOG.debug("%s was %s, converting to raw" % (image_id, fmt)) image_utils.convert_image(tmp, dest, 'raw') os.unlink(tmp) data = image_utils.qemu_img_info(dest) if data.file_format != "raw": msg = (_("Expected image to be in raw format, but is %s") % data.file_format) LOG.error(msg) raise exception.ImageUnacceptable( image_id=image_id, reason=msg) return {'size': math.ceil(data.virtual_size / 1024.0 ** 3)}
def fetch_verify_image(context, image_service, image_id, dest, user_id=None, project_id=None): fetch(context, image_service, image_id, dest, None, None) with fileutils.remove_path_on_error(dest): data = qemu_img_info(dest) fmt = data.file_format if fmt is None: raise exception.ImageUnacceptable( reason=_("'qemu-img info' parsing failed."), image_id=image_id) backing_file = data.backing_file if backing_file is not None: raise exception.ImageUnacceptable( image_id=image_id, reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file}))
def _gpfs_fetch(self, context, image_service, image_id, path, _user_id, _project_id): if not (self.configuration.gpfs_images_share_mode and self.configuration.gpfs_images_dir and os.path.exists( os.path.join(self.configuration.gpfs_images_dir, image_id))): with fileutils.remove_path_on_error(path): with open(path, "wb") as image_file: image_service.download(context, image_id, image_file) else: image_path = os.path.join(self.configuration.gpfs_images_dir, image_id) if self.configuration.gpfs_images_share_mode == 'copy_on_write': # check if the image is a GPFS snap file if not self._is_gpfs_parent_file(image_path): self._create_gpfs_snap(image_path, modebits='666') self._execute('ln', '-s', image_path, path, run_as_root=True) else: # copy self._execute('cp', image_path, path, run_as_root=True) self._execute('chmod', '666', path, run_as_root=True)
def fetch(context, image_service, image_id, path, _user_id, _project_id): # TODO(vish): Improve context handling and add owner and auth data # when it is added to glance. Right now there is no # auth checking in glance, so we assume that access was # checked before we got here. start_time = timeutils.utcnow() with fileutils.remove_path_on_error(path): with open(path, "wb") as image_file: image_service.download(context, image_id, image_file) duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in # some incredible event this is 0 (cirros image?) don't barf if duration < 1: duration = 1 fsz_mb = os.stat(image_file.name).st_size / units.Mi mbps = fsz_mb / duration msg = "Image fetch details: dest %(dest)s, size %(sz).2f MB, " "duration %(duration).2f sec" LOG.debug(msg % {"dest": image_file.name, "sz": fsz_mb, "duration": duration}) msg = _("Image download %(sz).2f MB at %(mbps).2f MB/s") LOG.info(msg % {"sz": fsz_mb, "mbps": mbps})
def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw', run_as_root=True): image_id = image_meta['id'] if (image_meta['disk_format'] == volume_format): LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta['disk_format'])) if os.name == 'nt' or os.access(volume_path, os.R_OK): with fileutils.file_open(volume_path, 'rb') as image_file: image_service.update(context, image_id, {}, image_file) else: with utils.temporary_chown(volume_path): with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta['disk_format'])) convert_image(volume_path, tmp, image_meta['disk_format'], bps_limit=CONF.volume_copy_bps_limit, run_as_root=run_as_root) data = qemu_img_info(tmp, run_as_root=run_as_root) if data.file_format != image_meta['disk_format']: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {'f1': image_meta['disk_format'], 'f2': data.file_format}) with fileutils.file_open(tmp, 'rb') as image_file: image_service.update(context, image_id, {}, image_file) fileutils.delete_if_exists(tmp)
def upload_volume(self, context, image_service, image_meta, volume_path): LOG.debug("Uploading volume %s: " ,volume_path) image_id = image_meta['id'] if (image_meta['disk_format'] == 'vhd'): LOG.debug("%s was raw, no need to convert to %s" % (image_id, image_meta['disk_format'])) with fileutils.file_open(volume_path) as image_file: image_service.update(context, image_id, {}, image_file) return if (CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir)): os.makedirs(CONF.image_conversion_dir) #Copy the volume to the image conversion dir temp_vhd_path = os.path.join(CONF.image_conversion_dir, str(image_meta['id']) + ".vhd") self.copy_vhd_disk(volume_path, temp_vhd_path) fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) os.close(fd) with fileutils.remove_path_on_error(tmp): LOG.debug("%s was vhd, converting to %s" % (image_id, image_meta['disk_format'])) self.convert_image(temp_vhd_path, tmp, image_meta['disk_format']) data = self.qemu_img_info(tmp) if data.file_format != image_meta['disk_format']: raise exception.ImageUnacceptable( image_id=image_id, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {'f1': image_meta['disk_format'], 'f2': data.file_format}) LOG.debug("Converted size of %s is: %s", data.backing_file, data.disk_size) with fileutils.file_open(tmp) as image_file: image_service.update(context, image_id, image_meta, image_file) os.unlink(tmp)