def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.basename(base)
# Copy main file of ploop disk, restore DiskDescriptor.xml for it
# and resize if necessary
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def _copy_ploop_image(base, target, size):
# Ploop disk is a directory with data file(root.hds) and
# DiskDescriptor.xml, so create this dir
fileutils.ensure_tree(target)
image_path = os.path.join(target, "root.hds")
libvirt_utils.copy_image(base, image_path)
utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
target, image_path)
if size:
self.resize_image(size)
# Generating means that we create empty ploop disk
generating = 'image_id' not in kwargs
remove_func = functools.partial(fileutils.delete_if_exists,
remove=shutil.rmtree)
if generating:
if os.path.exists(self.path):
return
with fileutils.remove_path_on_error(self.path, remove=remove_func):
prepare_template(target=self.path, *args, **kwargs)
else:
# Create ploop disk from glance image
if not os.path.exists(base):
prepare_template(target=base, *args, **kwargs)
else:
# Disk already exists in cache, just update time
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
if os.path.exists(self.path):
return
# Get format for ploop disk
if CONF.force_raw_images:
self.pcs_format = "raw"
else:
image_meta = IMAGE_API.get(kwargs["context"],
kwargs["image_id"])
format = image_meta.get("disk_format")
if format == "ploop":
self.pcs_format = "expanded"
elif format == "raw":
self.pcs_format = "raw"
else:
reason = _("Virtuozzo doesn't support images in %s format."
" You should either set force_raw_images=True"
" in config or upload an image in ploop"
" or raw format.") % format
raise exception.ImageUnacceptable(
image_id=kwargs["image_id"],
reason=reason)
with fileutils.remove_path_on_error(self.path, remove=remove_func):
_copy_ploop_image(base, self.path, size)
def test_update_mtime_error(self):
with mock.patch.object(libvirt_utils,
'execute',
side_effect=processutils.ProcessExecutionError):
with mock.patch.object(libvirt_utils.LOG, 'warning') as mock_log:
libvirt_utils.update_mtime(mock.sentinel.path)
self.assertTrue(mock_log.called)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.basename(base)
# Copy main file of ploop disk, restore DiskDescriptor.xml for it
# and resize if necessary
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def _copy_ploop_image(base, target, size):
# Ploop disk is a directory with data file(root.hds) and
# DiskDescriptor.xml, so create this dir
fileutils.ensure_tree(target)
image_path = os.path.join(target, "root.hds")
libvirt_utils.copy_image(base, image_path)
utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
target, image_path)
if size:
self.resize_image(size)
# Generating means that we create empty ploop disk
generating = 'image_id' not in kwargs
remove_func = functools.partial(fileutils.delete_if_exists,
remove=shutil.rmtree)
if generating:
if os.path.exists(self.path):
return
with fileutils.remove_path_on_error(self.path, remove=remove_func):
prepare_template(target=self.path, *args, **kwargs)
else:
# Create ploop disk from glance image
if not os.path.exists(base):
prepare_template(target=base, *args, **kwargs)
else:
# Disk already exists in cache, just update time
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
if os.path.exists(self.path):
return
# Get format for ploop disk
if CONF.force_raw_images:
self.pcs_format = "raw"
else:
image_meta = IMAGE_API.get(kwargs["context"],
kwargs["image_id"])
format = image_meta.get("disk_format")
if format == "ploop":
self.pcs_format = "expanded"
elif format == "raw":
self.pcs_format = "raw"
else:
reason = _("Ploop image backend doesn't support images in"
" %s format. You should either set"
" force_raw_images=True in config or upload an"
" image in ploop or raw format.") % format
raise exception.ImageUnacceptable(
image_id=kwargs["image_id"],
reason=reason)
with fileutils.remove_path_on_error(self.path, remove=remove_func):
_copy_ploop_image(base, self.path, size)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
# class Raw is misnamed, format may not be 'raw' in all cases
image = imgmodel.LocalFileImage(target,
self.driver_format)
disk.extend(image, size)
generating = 'image_id' not in kwargs
if generating:
if not self.check_image_exists():
# Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
# NOTE(mikal): Update the mtime of the base file so the image
# cache manager knows it is in use.
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def _handle_base_image(self, img_id, base_file):
"""Handle the checks for a single base image."""
image_in_use = False
LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), {
'id': img_id,
'base_file': base_file
})
if base_file in self.unexplained_images:
self.unexplained_images.remove(base_file)
if img_id in self.used_images:
local, remote, instances = self.used_images[img_id]
if local > 0 or remote > 0:
image_in_use = True
LOG.info(
_LI('image %(id)s at (%(base_file)s): '
'in use: on this node %(local)d local, '
'%(remote)d on other nodes sharing this instance '
'storage'), {
'id': img_id,
'base_file': base_file,
'local': local,
'remote': remote
})
self.active_base_files.append(base_file)
if not base_file:
LOG.warning(
_LW('image %(id)s at (%(base_file)s): warning '
'-- an absent base file is in use! '
'instances: %(instance_list)s'), {
'id': img_id,
'base_file': base_file,
'instance_list': ' '.join(instances)
})
if base_file:
if not image_in_use:
LOG.debug(
'image %(id)s at (%(base_file)s): image is not in '
'use', {
'id': img_id,
'base_file': base_file
})
self.removable_base_files.append(base_file)
else:
LOG.debug(
'image %(id)s at (%(base_file)s): image is in '
'use', {
'id': img_id,
'base_file': base_file
})
if os.path.exists(base_file):
libvirt_utils.update_mtime(base_file)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
if size:
image = imgmodel.LocalFileImage(target, self.driver_format)
disk.extend(image, size)
generating = 'image_id' not in kwargs
if generating:
if not self.exists():
# Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
if not os.path.exists(base):
prepare_template(target=base, *args, **kwargs)
# NOTE(mikal): Update the mtime of the base file so the image
# cache manager knows it is in use.
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
self.correct_format()
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
isuki = False
print 'whr [cr im] %s' % time.time()
#for key in kwargs:
# print "whr kwargs: %s: %s" % (key, kwargs[key])
# if key == 'isuki':
# isuki = True
isuki = 'isuki' in kwargs
time_ref = time.time()
time_prev = time_ref
if isuki:
print 'whr trueeeeee'
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_raw_image(base, target, size):
libvirt_utils.copy_image(base, target)
time_ref = time.time()
time_prev = time_ref
if size:
image = imgmodel.LocalFileImage(target,
self.driver_format)
disk.extend(image, size)
print 'whr [cr im] 0 %0.06f' % (time.time() - time_prev)
print 'whr [cr im] 1 %0.06f' % (time.time() - time_prev)
time_prev = time.time()
generating = 'image_id' not in kwargs
if generating:
if not self.exists():
print 'whr [cr im] 3 %0.06f' % (time.time() - time_prev)
time_prev = time.time()
# Generating image in place
prepare_template(target=self.path, *args, **kwargs)
else:
if not os.path.exists(base):
print 'whr [cr im] 4 %0.06f' % (time.time() - time_prev)
time_prev = time.time()
prepare_template(target=base, *args, **kwargs)
# NOTE(mikal): Update the mtime of the base file so the image
# cache manager knows it is in use.
if not isuki:
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
print 'whr [cr im] 5.0 %0.06f' % (time.time() - time_prev)
time_prev = time.time()
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_raw_image(base, self.path, size)
print 'whr [cr im] 5 %0.06f' % (time.time() - time_prev)
time_prev = time.time()
#self.correct_format()
print 'whr [cr im] 6 %0.06f' % (time.time() - time_prev)
time_prev = time.time()
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_qcow2_image(base, target, size):
# TODO(pbrady): Consider copying the cow image here
# with preallocation=metadata set for performance reasons.
# This would be keyed on a 'preallocate_images' setting.
#libvirt_utils.create_cow_image(base, target)
disk_format = kwargs.get('disk_format', '')
if disk_format == 'iso':
libvirt_utils.create_image('qcow2', target, size)
else:
libvirt_utils.create_cow_image(base, target)
if size:
image = imgmodel.LocalFileImage(target, imgmodel.FORMAT_QCOW2)
disk.extend(image, size)
# Download the unmodified base image unless we already have a copy.
if not os.path.exists(base):
prepare_template(target=base, *args, **kwargs)
# NOTE(ankit): Update the mtime of the base file so the image
# cache manager knows it is in use.
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
legacy_backing_size = None
legacy_base = base
# Determine whether an existing qcow2 disk uses a legacy backing by
# actually looking at the image itself and parsing the output of the
# backing file it expects to be using.
if os.path.exists(self.path):
backing_path = libvirt_utils.get_disk_backing_file(self.path)
if backing_path is not None:
backing_file = os.path.basename(backing_path)
backing_parts = backing_file.rpartition('_')
if backing_file != backing_parts[-1] and \
backing_parts[-1].isdigit():
legacy_backing_size = int(backing_parts[-1])
legacy_base += '_%d' % legacy_backing_size
legacy_backing_size *= units.Gi
# Create the legacy backing file if necessary.
if legacy_backing_size:
if not os.path.exists(legacy_base):
with fileutils.remove_path_on_error(legacy_base):
libvirt_utils.copy_image(base, legacy_base)
image = imgmodel.LocalFileImage(legacy_base,
imgmodel.FORMAT_QCOW2)
disk.extend(image, legacy_backing_size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_qcow2_image(base, self.path, size)
def _handle_base_image(self, img_id, base_file):
"""Handle the checks for a single base image."""
image_in_use = False
LOG.info(_LI('image %(id)s at (%(base_file)s): checking'),
{'id': img_id,
'base_file': base_file})
if base_file in self.unexplained_images:
self.unexplained_images.remove(base_file)
if img_id in self.used_images:
local, remote, instances = self.used_images[img_id]
if local > 0 or remote > 0:
image_in_use = True
LOG.info(_LI('image %(id)s at (%(base_file)s): '
'in use: on this node %(local)d local, '
'%(remote)d on other nodes sharing this instance '
'storage'),
{'id': img_id,
'base_file': base_file,
'local': local,
'remote': remote})
self.active_base_files.append(base_file)
if not base_file:
LOG.warning(_LW('image %(id)s at (%(base_file)s): warning '
'-- an absent base file is in use! '
'instances: %(instance_list)s'),
{'id': img_id,
'base_file': base_file,
'instance_list': ' '.join(instances)})
if base_file:
if not image_in_use:
LOG.debug('image %(id)s at (%(base_file)s): image is not in '
'use',
{'id': img_id,
'base_file': base_file})
self.removable_base_files.append(base_file)
else:
LOG.debug('image %(id)s at (%(base_file)s): image is in '
'use',
{'id': img_id,
'base_file': base_file})
if os.path.exists(base_file):
libvirt_utils.update_mtime(base_file)
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = self._get_lock_name(base)
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def copy_qcow2_image(base, target, size):
# TODO(pbrady): Consider copying the cow image here
# with preallocation=metadata set for performance reasons.
# This would be keyed on a 'preallocate_images' setting.
libvirt_utils.create_cow_image(base, target)
if size:
image = imgmodel.LocalFileImage(target, imgmodel.FORMAT_QCOW2)
disk.extend(image, size)
# Download the unmodified base image unless we already have a copy.
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
# NOTE(ankit): Update the mtime of the base file so the image
# cache manager knows it is in use.
libvirt_utils.update_mtime(base)
self.verify_base_size(base, size)
legacy_backing_size = None
legacy_base = base
# Determine whether an existing qcow2 disk uses a legacy backing by
# actually looking at the image itself and parsing the output of the
# backing file it expects to be using.
if os.path.exists(self.path):
backing_path = libvirt_utils.get_disk_backing_file(self.path)
if backing_path is not None:
backing_file = os.path.basename(backing_path)
backing_parts = backing_file.rpartition('_')
if backing_file != backing_parts[-1] and \
backing_parts[-1].isdigit():
legacy_backing_size = int(backing_parts[-1])
legacy_base += '_%d' % legacy_backing_size
legacy_backing_size *= units.Gi
# Create the legacy backing file if necessary.
if legacy_backing_size:
if not os.path.exists(legacy_base):
with fileutils.remove_path_on_error(legacy_base):
libvirt_utils.copy_image(base, legacy_base)
image = imgmodel.LocalFileImage(legacy_base,
imgmodel.FORMAT_QCOW2)
disk.extend(image, legacy_backing_size)
if not os.path.exists(self.path):
with fileutils.remove_path_on_error(self.path):
copy_qcow2_image(base, self.path, size)
def _mark_in_use(self, img_id, base_file):
"""Mark a single base image as in use."""
LOG.info(_LI('image %(id)s at (%(base_file)s): checking'),
{'id': img_id, 'base_file': base_file})
if base_file in self.unexplained_images:
self.unexplained_images.remove(base_file)
self.active_base_files.append(base_file)
LOG.debug('image %(id)s at (%(base_file)s): image is in use',
{'id': img_id, 'base_file': base_file})
libvirt_utils.update_mtime(base_file)
def _age_and_verify_swap_images(self, context, base_dir):
LOG.debug('Verify swap images')
for ent in self.back_swap_images:
base_file = os.path.join(base_dir, ent)
if ent in self.used_swap_images and os.path.exists(base_file):
libvirt_utils.update_mtime(base_file)
elif self.remove_unused_base_images:
self._remove_swap_file(base_file)
error_images = self.used_swap_images - self.back_swap_images
for error_image in error_images:
LOG.warn(_LW('%s swap image was used by instance'
' but no back files existing!'), error_image)
def _age_and_verify_swap_images(self, context, base_dir):
LOG.debug('Verify swap images')
for ent in self.back_swap_images:
base_file = os.path.join(base_dir, ent)
if ent in self.used_swap_images and os.path.exists(base_file):
libvirt_utils.update_mtime(base_file)
elif self.remove_unused_base_images:
self._remove_swap_file(base_file)
error_images = self.used_swap_images - self.back_swap_images
for error_image in error_images:
LOG.warning(_LW('%s swap image was used by instance'
' but no back files existing!'), error_image)
def _mark_in_use(self, img_id, base_file):
"""Mark a single base image as in use."""
LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), {
'id': img_id,
'base_file': base_file
})
if base_file in self.unexplained_images:
self.unexplained_images.remove(base_file)
self.active_base_files.append(base_file)
LOG.debug('image %(id)s at (%(base_file)s): image is in use', {
'id': img_id,
'base_file': base_file
})
libvirt_utils.update_mtime(base_file)
def test_update_mtime_error(self):
with mock.patch.object(libvirt_utils, 'execute',
side_effect=processutils.ProcessExecutionError):
with mock.patch.object(libvirt_utils.LOG, 'warning') as mock_log:
libvirt_utils.update_mtime(mock.sentinel.path)
self.assertTrue(mock_log.called)
def _handle_base_image(self, img_id, base_file):
"""Handle the checks for a single base image."""
image_bad = False
image_in_use = False
LOG.info(_LI('image %(id)s at (%(base_file)s): checking'),
{'id': img_id,
'base_file': base_file})
if base_file in self.unexplained_images:
self.unexplained_images.remove(base_file)
if (base_file and os.path.exists(base_file)
and os.path.isfile(base_file)):
# _verify_checksum returns True if the checksum is ok, and None if
# there is no checksum file
checksum_result = self._verify_checksum(img_id, base_file)
if checksum_result is not None:
image_bad = not checksum_result
# Give other threads a chance to run
time.sleep(0)
if img_id in self.used_images:
local, remote, instances = self.used_images[img_id]
if local > 0 or remote > 0:
image_in_use = True
LOG.info(_LI('image %(id)s at (%(base_file)s): '
'in use: on this node %(local)d local, '
'%(remote)d on other nodes sharing this instance '
'storage'),
{'id': img_id,
'base_file': base_file,
'local': local,
'remote': remote})
self.active_base_files.append(base_file)
if not base_file:
LOG.warn(_LW('image %(id)s at (%(base_file)s): warning '
'-- an absent base file is in use! '
'instances: %(instance_list)s'),
{'id': img_id,
'base_file': base_file,
'instance_list': ' '.join(instances)})
if image_bad:
self.corrupt_base_files.append(base_file)
if base_file:
if not image_in_use:
LOG.debug('image %(id)s at (%(base_file)s): image is not in '
'use',
{'id': img_id,
'base_file': base_file})
self.removable_base_files.append(base_file)
else:
LOG.debug('image %(id)s at (%(base_file)s): image is in '
'use',
{'id': img_id,
'base_file': base_file})
if os.path.exists(base_file):
libvirt_utils.update_mtime(base_file)
def _handle_base_image(self, img_id, base_file):
"""Handle the checks for a single base image."""
image_bad = False
image_in_use = False
LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), {
'id': img_id,
'base_file': base_file
})
if base_file in self.unexplained_images:
self.unexplained_images.remove(base_file)
if (base_file and os.path.exists(base_file)
and os.path.isfile(base_file)):
# _verify_checksum returns True if the checksum is ok, and None if
# there is no checksum file
checksum_result = self._verify_checksum(img_id, base_file)
if checksum_result is not None:
image_bad = not checksum_result
# Give other threads a chance to run
time.sleep(0)
if img_id in self.used_images:
local, remote, instances = self.used_images[img_id]
if local > 0 or remote > 0:
image_in_use = True
LOG.info(
_LI('image %(id)s at (%(base_file)s): '
'in use: on this node %(local)d local, '
'%(remote)d on other nodes sharing this instance '
'storage'), {
'id': img_id,
'base_file': base_file,
'local': local,
'remote': remote
})
self.active_base_files.append(base_file)
if not base_file:
LOG.warning(
_LW('image %(id)s at (%(base_file)s): warning '
'-- an absent base file is in use! '
'instances: %(instance_list)s'), {
'id': img_id,
'base_file': base_file,
'instance_list': ' '.join(instances)
})
if image_bad:
self.corrupt_base_files.append(base_file)
if base_file:
if not image_in_use:
LOG.debug(
'image %(id)s at (%(base_file)s): image is not in '
'use', {
'id': img_id,
'base_file': base_file
})
self.removable_base_files.append(base_file)
else:
LOG.debug(
'image %(id)s at (%(base_file)s): image is in '
'use', {
'id': img_id,
'base_file': base_file
})
if os.path.exists(base_file):
libvirt_utils.update_mtime(base_file)
