def fetch_to_raw(context, image_href, path, user_id, project_id):
path_tmp = "%s.part" % path
fetch(context, image_href, path_tmp, user_id, project_id)
with fileutils.remove_path_on_error(path_tmp):
data = qemu_img_info(path_tmp)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_href)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(image_id=image_href,
reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") %
{'fmt': fmt, 'backing_file': backing_file}))
if fmt != "raw" and CONF.force_raw_images:
staged = "%s.converted" % path
LOG.debug("%s was %s, converting to raw" % (image_href, fmt))
with fileutils.remove_path_on_error(staged):
convert_image(path_tmp, staged, 'raw')
os.unlink(path_tmp)
data = qemu_img_info(staged)
if data.file_format != "raw":
raise exception.ImageUnacceptable(image_id=image_href,
reason=_("Converted to raw, but format is now %s") %
data.file_format)
os.rename(staged, path)
else:
os.rename(path_tmp, path)
def fetch_and_convert(context, image_href, path, image_fmt):
path_tmp = "%s.part" % path
#fetch image
with fileutils.remove_path_on_error(path_tmp):
IMAGE_API.download(context, image_href, dest_path=path_tmp)
#convert image
with fileutils.remove_path_on_error(path_tmp):
data = qemu_img_info(path_tmp)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_href)
if fmt != image_fmt:
staged = "%s.converted" % path
with fileutils.remove_path_on_error(staged):
convert_image(path_tmp, staged, image_fmt)
os.unlink(path_tmp)
data = qemu_img_info(staged)
if data.file_format != image_fmt:
raise exception.ImageUnacceptable(image_id=image_href,
reason=_("Converted to %s, but format is now %s") %
(image_fmt, data.file_format))
os.rename(staged, path)
else:
os.rename(path_tmp, path)
def fetch_to_raw(context, image_href, path, user_id, project_id):
path_tmp = "%s.part" % path
fetch(context, image_href, path_tmp, user_id, project_id)
with fileutils.remove_path_on_error(path_tmp):
data = qemu_img_info(path_tmp)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_href)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(image_id=image_href,
reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") %
{'fmt': fmt, 'backing_file': backing_file}))
if fmt != "raw" and CONF.force_raw_images:
staged = "%s.converted" % path
LOG.debug("%s was %s, converting to raw" % (image_href, fmt))
with fileutils.remove_path_on_error(staged):
convert_image(path_tmp, staged, 'raw')
os.unlink(path_tmp)
data = qemu_img_info(staged)
if data.file_format != "raw":
raise exception.ImageUnacceptable(image_id=image_href,
reason=_("Converted to raw, but format is now %s") %
data.file_format)
os.rename(staged, path)
else:
os.rename(path_tmp, path)
def create_image(self, prepare_template, base, size, *args, **kwargs):
@utils.synchronized(base, external=True, lock_path=self.lock_path)
def copy_qcow2_image(base, target, size):
# TODO(pbrady): Consider copying the cow image here
# with preallocation=metadata set for performance reasons.
# This would be keyed on a 'preallocate_images' setting.
libvirt_utils.create_cow_image(base, target)
if size:
disk.extend(target, size)
# Download the unmodified base image unless we already have a copy.
if not os.path.exists(base):
prepare_template(target=base, *args, **kwargs)
legacy_backing_size = None
legacy_base = base
# Determine whether an existing qcow2 disk uses a legacy backing by
# actually looking at the image itself and parsing the output of the
# backing file it expects to be using.
if os.path.exists(self.path):
backing_path = libvirt_utils.get_disk_backing_file(self.path)
if backing_path is not None:
backing_file = os.path.basename(backing_path)
backing_parts = backing_file.rpartition('_')
if backing_file != backing_parts[-1] and \
backing_parts[-1].isdigit():
legacy_backing_size = int(backing_parts[-1])
legacy_base += '_%d' % legacy_backing_size
legacy_backing_size *= 1024 * 1024 * 1024
# Create the legacy backing file if necessary.
if legacy_backing_size:
if not os.path.exists(legacy_base):
with fileutils.remove_path_on_error(legacy_base):
libvirt_utils.copy_image(base, legacy_base)
disk.extend(legacy_base, legacy_backing_size)
# NOTE(cfb): Having a flavor that sets the root size to 0 and having
# nova effectively ignore that size and use the size of the
# image is considered a feature at this time, not a bug.
disk_size = disk.get_disk_size(base)
if size and size < disk_size:
msg = _('%(base)s virtual size %(disk_size)s'
'larger than flavor root disk size %(size)s')
LOG.error(msg % {
'base': base,
'disk_size': disk_size,
'size': size
})
raise exception.InstanceTypeDiskTooSmall()
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_qcow2_image(base, self.path, size)
def fetch_to_raw(context, image_href, path, user_id, project_id, max_size=0):
path_tmp = "%s.part" % path
fetch(context, image_href, path_tmp, user_id, project_id,
max_size=max_size)
with fileutils.remove_path_on_error(path_tmp):
data = qemu_img_info(path_tmp)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_href)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(image_id=image_href,
reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") %
{'fmt': fmt, 'backing_file': backing_file}))
# We can't generally shrink incoming images, so disallow
# images > size of the flavor we're booting. Checking here avoids
# an immediate DoS where we convert large qcow images to raw
# (which may compress well but not be sparse).
# TODO(p-draigbrady): loop through all flavor sizes, so that
# we might continue here and not discard the download.
# If we did that we'd have to do the higher level size checks
# irrespective of whether the base image was prepared or not.
disk_size = data.virtual_size
if max_size and max_size < disk_size:
LOG.error(_LE('%(base)s virtual size %(disk_size)s '
'larger than flavor root disk size %(size)s'),
{'base': path,
'disk_size': disk_size,
'size': max_size})
raise exception.FlavorDiskTooSmall()
if fmt != "raw" and CONF.force_raw_images:
staged = "%s.converted" % path
LOG.debug("%s was %s, converting to raw" % (image_href, fmt))
with fileutils.remove_path_on_error(staged):
convert_image(path_tmp, staged, 'raw')
os.unlink(path_tmp)
data = qemu_img_info(staged)
if data.file_format != "raw":
raise exception.ImageUnacceptable(image_id=image_href,
reason=_("Converted to raw, but format is now %s") %
data.file_format)
os.rename(staged, path)
else:
os.rename(path_tmp, path)
def fetch_to_raw(context, image_href, path, user_id, project_id, max_size=0):
path_tmp = "%s.part" % path
fetch(context, image_href, path_tmp, user_id, project_id,
max_size=max_size)
with fileutils.remove_path_on_error(path_tmp):
data = qemu_img_info(path_tmp)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_href)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(image_id=image_href,
reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") %
{'fmt': fmt, 'backing_file': backing_file}))
# We can't generally shrink incoming images, so disallow
# images > size of the flavor we're booting. Checking here avoids
# an immediate DoS where we convert large qcow images to raw
# (which may compress well but not be sparse).
# TODO(p-draigbrady): loop through all flavor sizes, so that
# we might continue here and not discard the download.
# If we did that we'd have to do the higher level size checks
# irrespective of whether the base image was prepared or not.
disk_size = data.virtual_size
if max_size and max_size < disk_size:
msg = _('%(base)s virtual size %(disk_size)s '
'larger than flavor root disk size %(size)s')
LOG.error(msg % {'base': path,
'disk_size': disk_size,
'size': max_size})
raise exception.FlavorDiskTooSmall()
if fmt != "raw" and CONF.force_raw_images:
staged = "%s.converted" % path
LOG.debug("%s was %s, converting to raw" % (image_href, fmt))
with fileutils.remove_path_on_error(staged):
convert_image(path_tmp, staged, 'raw')
os.unlink(path_tmp)
data = qemu_img_info(staged)
if data.file_format != "raw":
raise exception.ImageUnacceptable(image_id=image_href,
reason=_("Converted to raw, but format is now %s") %
data.file_format)
os.rename(staged, path)
else:
os.rename(path_tmp, path)
def create_image(self, prepare_template, base, size, *args, **kwargs):
@utils.synchronized(base, external=True, lock_path=self.lock_path)
def copy_qcow2_image(base, target, size):
# TODO(pbrady): Consider copying the cow image here
# with preallocation=metadata set for performance reasons.
# This would be keyed on a 'preallocate_images' setting.
libvirt_utils.create_cow_image(base, target)
if size:
disk.extend(target, size, use_cow=True)
# Download the unmodified base image unless we already have a copy.
if not os.path.exists(base):
prepare_template(target=base, *args, **kwargs)
legacy_backing_size = None
legacy_base = base
# Determine whether an existing qcow2 disk uses a legacy backing by
# actually looking at the image itself and parsing the output of the
# backing file it expects to be using.
if os.path.exists(self.path):
backing_path = libvirt_utils.get_disk_backing_file(self.path)
if backing_path is not None:
backing_file = os.path.basename(backing_path)
backing_parts = backing_file.rpartition('_')
if backing_file != backing_parts[-1] and \
backing_parts[-1].isdigit():
legacy_backing_size = int(backing_parts[-1])
legacy_base += '_%d' % legacy_backing_size
legacy_backing_size *= 1024 * 1024 * 1024
# Create the legacy backing file if necessary.
if legacy_backing_size:
if not os.path.exists(legacy_base):
with fileutils.remove_path_on_error(legacy_base):
libvirt_utils.copy_image(base, legacy_base)
disk.extend(legacy_base, legacy_backing_size, use_cow=True)
# NOTE(cfb): Having a flavor that sets the root size to 0 and having
# nova effectively ignore that size and use the size of the
# image is considered a feature at this time, not a bug.
disk_size = disk.get_disk_size(base)
if size and size < disk_size:
msg = _('%(base)s virtual size %(disk_size)s'
'larger than flavor root disk size %(size)s')
LOG.error(msg % {'base': base,
'disk_size': disk_size,
'size': size})
raise exception.InstanceTypeDiskTooSmall()
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_qcow2_image(base, self.path, size)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
# class Raw is misnamed, format may not be 'raw' in all cases
image = imgmodel.LocalFileImage(target,
self.driver_format)
disk.extend(image, size)
generating = 'image_id' not in kwargs
if generating:
if not self.check_image_exists():
# Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_qcow2_image(base, target, size):
# TODO(pbrady): Consider copying the cow image here
# with preallocation=metadata set for performance reasons.
# This would be keyed on a 'preallocate_images' setting.
libvirt_utils.create_cow_image(base, target)
if size:
image = imgmodel.LocalFileImage(target, imgmodel.FORMAT_QCOW2)
disk.extend(image, size)
# Download the unmodified base image unless we already have a copy.
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
legacy_backing_size = None
legacy_base = base
# Determine whether an existing qcow2 disk uses a legacy backing by
# actually looking at the image itself and parsing the output of the
# backing file it expects to be using.
if os.path.exists(self.path):
backing_path = libvirt_utils.get_disk_backing_file(self.path)
if backing_path is not None:
backing_file = os.path.basename(backing_path)
backing_parts = backing_file.rpartition('_')
if backing_file != backing_parts[-1] and \
backing_parts[-1].isdigit():
legacy_backing_size = int(backing_parts[-1])
legacy_base += '_%d' % legacy_backing_size
legacy_backing_size *= units.Gi
# Create the legacy backing file if necessary.
if legacy_backing_size:
if not os.path.exists(legacy_base):
with fileutils.remove_path_on_error(legacy_base):
libvirt_utils.copy_image(base, legacy_base)
image = imgmodel.LocalFileImage(legacy_base,
imgmodel.FORMAT_QCOW2)
disk.extend(image, legacy_backing_size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_qcow2_image(base, self.path, size)
def fetch(context, image_href, path, _user_id, _project_id, max_size=0):
# TODO(vish): Improve context handling and add owner and auth data
# when it is added to glance. Right now there is no
# auth checking in glance, so we assume that access was
# checked before we got here.
(image_service, image_id) = glance.get_remote_image_service(context, image_href)
with fileutils.remove_path_on_error(path):
image_service.download(context, image_id, dst_path=path)
def fetch(context, image_href, path, _user_id, _project_id, max_size=0):
# TODO(vish): Improve context handling and add owner and auth data
# when it is added to glance. Right now there is no
# auth checking in glance, so we assume that access was
# checked before we got here.
(image_service,
image_id) = glance.get_remote_image_service(context, image_href)
with fileutils.remove_path_on_error(path):
image_service.download(context, image_id, dst_path=path)
def _fetch_image(self, context, image_id, image_meta, path,
user_id=None, project_id=None, location=None,
**kwargs):
# TODO(vish): Improve context handling and add owner and auth data
# when it is added to glance. Right now there is no
# auth checking in glance, so we assume that access was
# checked before we got here.
(image_service, _image_id) = glance.get_remote_image_service(context,
image_id)
with fileutils.remove_path_on_error(path):
image_service.download(context, image_id, dst_path=path)
return os.path.exists(path)
def fetch_to_qcow2(context, image_href,path, user_id, project_id, max_size=0):
path_tmp = "%s.part" % path
fetch(context, image_href, path_tmp, user_id, project_id,
max_size=max_size)
with fileutils.remove_path_on_error(path_tmp):
data = qemu_img_info(path_tmp)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."),
image_id=image_href)
disk_size = data.virtual_size
if max_size and max_size < disk_size:
LOG.error(_LE('%(base)s virtual size %(disk_size)s '
'larger than flavor root disk size %(size)s'),
{'base': path,
'disk_size': disk_size,
'size': max_size})
raise exception.FlavorDiskTooSmall()
if fmt != "qcow2":
staged = "%s.converted" % path
LOG.debug("%s was %s, converting to qcow2" % (image_href, fmt))
with fileutils.remove_path_on_error(staged):
convert_image(path_tmp, staged, 'qcow2')
os.unlink(path_tmp)
data = qemu_img_info(staged)
if data.file_format != "qcow2":
raise exception.ImageUnacceptable(image_id=image_href,
reason=_("Converted to qcow2, but format is now %s") %
data.file_format)
os.rename(staged, path)
else:
os.rename(path_tmp, path)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.split(base)[-1]
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def create_ploop_image(base, target, size):
image_path = os.path.join(target, "root.hds")
libvirt_utils.copy_image(base, image_path)
utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
target, image_path)
if size:
dd_path = os.path.join(self.path, "DiskDescriptor.xml")
utils.execute('ploop',
'grow',
'-s',
'%dK' % (size >> 10),
dd_path,
run_as_root=True)
if not os.path.exists(self.path):
if CONF.force_raw_images:
self.pcs_format = "raw"
else:
image_meta = IMAGE_API.get(kwargs["context"],
kwargs["image_id"])
format = image_meta.get("disk_format")
if format == "ploop":
self.pcs_format = "expanded"
elif format == "raw":
self.pcs_format = "raw"
else:
reason = _("PCS doesn't support images in %s format."
" You should either set force_raw_images=True"
" in config or upload an image in ploop"
" or raw format.") % format
raise exception.ImageUnacceptable(
image_id=kwargs["image_id"], reason=reason)
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
if os.path.exists(self.path):
return
fileutils.ensure_tree(self.path)
remove_func = functools.partial(fileutils.delete_if_exists,
remove=shutil.rmtree)
with fileutils.remove_path_on_error(self.path, remove=remove_func):
create_ploop_image(base, self.path, size)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.split(base)[-1]
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def create_ploop_image(base, target, size):
image_path = os.path.join(target, "root.hds")
libvirt_utils.copy_image(base, image_path)
utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
target, image_path)
if size:
dd_path = os.path.join(self.path, "DiskDescriptor.xml")
utils.execute('ploop', 'grow', '-s', '%dK' % (size >> 10),
dd_path, run_as_root=True)
if not os.path.exists(self.path):
if CONF.force_raw_images:
self.pcs_format = "raw"
else:
image_meta = IMAGE_API.get(kwargs["context"],
kwargs["image_id"])
format = image_meta.get("disk_format")
if format == "ploop":
self.pcs_format = "expanded"
elif format == "raw":
self.pcs_format = "raw"
else:
reason = _("PCS doesn't support images in %s format."
" You should either set force_raw_images=True"
" in config or upload an image in ploop"
" or raw format.") % format
raise exception.ImageUnacceptable(
image_id=kwargs["image_id"],
reason=reason)
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
if os.path.exists(self.path):
return
fileutils.ensure_tree(self.path)
remove_func = functools.partial(fileutils.delete_if_exists,
remove=shutil.rmtree)
with fileutils.remove_path_on_error(self.path, remove=remove_func):
create_ploop_image(base, self.path, size)
def create_image(self, prepare_template, base, size, *args, **kwargs):
@utils.synchronized(base, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
disk.extend(target, size)
generating = 'image_id' not in kwargs
if generating:
#Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
prepare_template(target=base, *args, **kwargs)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def create_image(self, prepare_template, base, size, *args, **kwargs):
@utils.synchronized(base, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
disk.extend(target, size)
generating = 'image_id' not in kwargs
if generating:
#Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
prepare_template(target=base, *args, **kwargs)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def create_image(self, prepare_template, base, size, *args, **kwargs):
@utils.synchronized(base, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
# class Raw is misnamed, format may not be 'raw' in all cases
use_cow = self.driver_format == 'qcow2'
disk.extend(target, size, use_cow=use_cow)
generating = 'image_id' not in kwargs
if generating:
#Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
prepare_template(target=base, *args, **kwargs)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def fetch(context, image_href, path, _user_id, _project_id,
max_size=0, imagehandler_args=None):
"""Fetch image and returns whether the image was stored locally."""
imagehandler_args = imagehandler_args or {}
_remove_image_fun = functools.partial(_remove_image_on_exec,
context, image_href,
_user_id, _project_id,
imagehandler_args)
fetched_to_local = True
with fileutils.remove_path_on_error(path, remove=_remove_image_fun):
for handler, loc, image_meta in imagehandler.handle_image(context,
image_href,
_user_id,
_project_id,
path):
# The loop will stop when the handle function returns success.
handler.fetch_image(context, image_href, image_meta, path,
_user_id, _project_id, loc,
**imagehandler_args)
fetched_to_local = handler.is_local()
return fetched_to_local
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.split(base)[-1]
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
# class Raw is misnamed, format may not be 'raw' in all cases
use_cow = self.driver_format == "qcow2"
disk.extend(target, size, use_cow=use_cow)
generating = "image_id" not in kwargs
if generating:
if not self.check_image_exists():
# Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def fetch(context, image_href, path, _user_id, _project_id, max_size=0):
with fileutils.remove_path_on_error(path):
IMAGE_API.download(context, image_href, dest_path=path)
def fetch(context, image_href, path, _user_id, _project_id, max_size=0):
with fileutils.remove_path_on_error(path):
IMAGE_API.download(context, image_href, dest_path=path)
