def create_image(self, prepare_template, base, size, *args, **kwargs): filename = os.path.basename(base) # Copy main file of ploop disk, restore DiskDescriptor.xml for it # and resize if necessary @utils.synchronized(filename, external=True, lock_path=self.lock_path) def _copy_ploop_image(base, target, size): # Ploop disk is a directory with data file(root.hds) and # DiskDescriptor.xml, so create this dir fileutils.ensure_tree(target) image_path = os.path.join(target, "root.hds") libvirt_utils.copy_image(base, image_path) utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format, target, image_path) if size: self.resize_image(size) # Generating means that we create empty ploop disk generating = 'image_id' not in kwargs remove_func = functools.partial(fileutils.delete_if_exists, remove=shutil.rmtree) if generating: if os.path.exists(self.path): return with fileutils.remove_path_on_error(self.path, remove=remove_func): prepare_template(target=self.path, *args, **kwargs) else: # Create ploop disk from glance image if not os.path.exists(base): prepare_template(target=base, *args, **kwargs) else: # Disk already exists in cache, just update time libvirt_utils.update_mtime(base) self.verify_base_size(base, size) if os.path.exists(self.path): return # Get format for ploop disk if CONF.force_raw_images: self.pcs_format = "raw" else: image_meta = IMAGE_API.get(kwargs["context"], kwargs["image_id"]) format = image_meta.get("disk_format") if format == "ploop": self.pcs_format = "expanded" elif format == "raw": self.pcs_format = "raw" else: reason = _("Virtuozzo doesn't support images in %s format." " You should either set force_raw_images=True" " in config or upload an image in ploop" " or raw format.") % format raise exception.ImageUnacceptable( image_id=kwargs["image_id"], reason=reason) with fileutils.remove_path_on_error(self.path, remove=remove_func): _copy_ploop_image(base, self.path, size)
def test_update_mtime_error(self): with mock.patch.object(libvirt_utils, 'execute', side_effect=processutils.ProcessExecutionError): with mock.patch.object(libvirt_utils.LOG, 'warning') as mock_log: libvirt_utils.update_mtime(mock.sentinel.path) self.assertTrue(mock_log.called)
def create_image(self, prepare_template, base, size, *args, **kwargs): filename = os.path.basename(base) # Copy main file of ploop disk, restore DiskDescriptor.xml for it # and resize if necessary @utils.synchronized(filename, external=True, lock_path=self.lock_path) def _copy_ploop_image(base, target, size): # Ploop disk is a directory with data file(root.hds) and # DiskDescriptor.xml, so create this dir fileutils.ensure_tree(target) image_path = os.path.join(target, "root.hds") libvirt_utils.copy_image(base, image_path) utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format, target, image_path) if size: self.resize_image(size) # Generating means that we create empty ploop disk generating = 'image_id' not in kwargs remove_func = functools.partial(fileutils.delete_if_exists, remove=shutil.rmtree) if generating: if os.path.exists(self.path): return with fileutils.remove_path_on_error(self.path, remove=remove_func): prepare_template(target=self.path, *args, **kwargs) else: # Create ploop disk from glance image if not os.path.exists(base): prepare_template(target=base, *args, **kwargs) else: # Disk already exists in cache, just update time libvirt_utils.update_mtime(base) self.verify_base_size(base, size) if os.path.exists(self.path): return # Get format for ploop disk if CONF.force_raw_images: self.pcs_format = "raw" else: image_meta = IMAGE_API.get(kwargs["context"], kwargs["image_id"]) format = image_meta.get("disk_format") if format == "ploop": self.pcs_format = "expanded" elif format == "raw": self.pcs_format = "raw" else: reason = _("Ploop image backend doesn't support images in" " %s format. You should either set" " force_raw_images=True in config or upload an" " image in ploop or raw format.") % format raise exception.ImageUnacceptable( image_id=kwargs["image_id"], reason=reason) with fileutils.remove_path_on_error(self.path, remove=remove_func): _copy_ploop_image(base, self.path, size)
def create_image(self, prepare_template, base, size, *args, **kwargs): filename = self._get_lock_name(base) @utils.synchronized(filename, external=True, lock_path=self.lock_path) def copy_raw_image(base, target, size): libvirt_utils.copy_image(base, target) if size: # class Raw is misnamed, format may not be 'raw' in all cases image = imgmodel.LocalFileImage(target, self.driver_format) disk.extend(image, size) generating = 'image_id' not in kwargs if generating: if not self.check_image_exists(): # Generating image in place prepare_template(target=self.path, *args, **kwargs) else: if not os.path.exists(base): prepare_template(target=base, max_size=size, *args, **kwargs) # NOTE(mikal): Update the mtime of the base file so the image # cache manager knows it is in use. libvirt_utils.update_mtime(base) self.verify_base_size(base, size) if not os.path.exists(self.path): with fileutils.remove_path_on_error(self.path): copy_raw_image(base, self.path, size) self.correct_format()
def _handle_base_image(self, img_id, base_file): """Handle the checks for a single base image.""" image_in_use = False LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), { 'id': img_id, 'base_file': base_file }) if base_file in self.unexplained_images: self.unexplained_images.remove(base_file) if img_id in self.used_images: local, remote, instances = self.used_images[img_id] if local > 0 or remote > 0: image_in_use = True LOG.info( _LI('image %(id)s at (%(base_file)s): ' 'in use: on this node %(local)d local, ' '%(remote)d on other nodes sharing this instance ' 'storage'), { 'id': img_id, 'base_file': base_file, 'local': local, 'remote': remote }) self.active_base_files.append(base_file) if not base_file: LOG.warning( _LW('image %(id)s at (%(base_file)s): warning ' '-- an absent base file is in use! ' 'instances: %(instance_list)s'), { 'id': img_id, 'base_file': base_file, 'instance_list': ' '.join(instances) }) if base_file: if not image_in_use: LOG.debug( 'image %(id)s at (%(base_file)s): image is not in ' 'use', { 'id': img_id, 'base_file': base_file }) self.removable_base_files.append(base_file) else: LOG.debug( 'image %(id)s at (%(base_file)s): image is in ' 'use', { 'id': img_id, 'base_file': base_file }) if os.path.exists(base_file): libvirt_utils.update_mtime(base_file)
def create_image(self, prepare_template, base, size, *args, **kwargs): filename = self._get_lock_name(base) @utils.synchronized(filename, external=True, lock_path=self.lock_path) def copy_raw_image(base, target, size): libvirt_utils.copy_image(base, target) if size: image = imgmodel.LocalFileImage(target, self.driver_format) disk.extend(image, size) generating = 'image_id' not in kwargs if generating: if not self.exists(): # Generating image in place prepare_template(target=self.path, *args, **kwargs) else: if not os.path.exists(base): prepare_template(target=base, *args, **kwargs) # NOTE(mikal): Update the mtime of the base file so the image # cache manager knows it is in use. libvirt_utils.update_mtime(base) self.verify_base_size(base, size) if not os.path.exists(self.path): with fileutils.remove_path_on_error(self.path): copy_raw_image(base, self.path, size) self.correct_format()
def create_image(self, prepare_template, base, size, *args, **kwargs): filename = self._get_lock_name(base) isuki = False print 'whr [cr im] %s' % time.time() #for key in kwargs: # print "whr kwargs: %s: %s" % (key, kwargs[key]) # if key == 'isuki': # isuki = True isuki = 'isuki' in kwargs time_ref = time.time() time_prev = time_ref if isuki: print 'whr trueeeeee' @utils.synchronized(filename, external=True, lock_path=self.lock_path) def copy_raw_image(base, target, size): libvirt_utils.copy_image(base, target) time_ref = time.time() time_prev = time_ref if size: image = imgmodel.LocalFileImage(target, self.driver_format) disk.extend(image, size) print 'whr [cr im] 0 %0.06f' % (time.time() - time_prev) print 'whr [cr im] 1 %0.06f' % (time.time() - time_prev) time_prev = time.time() generating = 'image_id' not in kwargs if generating: if not self.exists(): print 'whr [cr im] 3 %0.06f' % (time.time() - time_prev) time_prev = time.time() # Generating image in place prepare_template(target=self.path, *args, **kwargs) else: if not os.path.exists(base): print 'whr [cr im] 4 %0.06f' % (time.time() - time_prev) time_prev = time.time() prepare_template(target=base, *args, **kwargs) # NOTE(mikal): Update the mtime of the base file so the image # cache manager knows it is in use. if not isuki: libvirt_utils.update_mtime(base) self.verify_base_size(base, size) print 'whr [cr im] 5.0 %0.06f' % (time.time() - time_prev) time_prev = time.time() if not os.path.exists(self.path): with fileutils.remove_path_on_error(self.path): copy_raw_image(base, self.path, size) print 'whr [cr im] 5 %0.06f' % (time.time() - time_prev) time_prev = time.time() #self.correct_format() print 'whr [cr im] 6 %0.06f' % (time.time() - time_prev) time_prev = time.time()
def create_image(self, prepare_template, base, size, *args, **kwargs): filename = self._get_lock_name(base) @utils.synchronized(filename, external=True, lock_path=self.lock_path) def copy_qcow2_image(base, target, size): # TODO(pbrady): Consider copying the cow image here # with preallocation=metadata set for performance reasons. # This would be keyed on a 'preallocate_images' setting. #libvirt_utils.create_cow_image(base, target) disk_format = kwargs.get('disk_format', '') if disk_format == 'iso': libvirt_utils.create_image('qcow2', target, size) else: libvirt_utils.create_cow_image(base, target) if size: image = imgmodel.LocalFileImage(target, imgmodel.FORMAT_QCOW2) disk.extend(image, size) # Download the unmodified base image unless we already have a copy. if not os.path.exists(base): prepare_template(target=base, *args, **kwargs) # NOTE(ankit): Update the mtime of the base file so the image # cache manager knows it is in use. libvirt_utils.update_mtime(base) self.verify_base_size(base, size) legacy_backing_size = None legacy_base = base # Determine whether an existing qcow2 disk uses a legacy backing by # actually looking at the image itself and parsing the output of the # backing file it expects to be using. if os.path.exists(self.path): backing_path = libvirt_utils.get_disk_backing_file(self.path) if backing_path is not None: backing_file = os.path.basename(backing_path) backing_parts = backing_file.rpartition('_') if backing_file != backing_parts[-1] and \ backing_parts[-1].isdigit(): legacy_backing_size = int(backing_parts[-1]) legacy_base += '_%d' % legacy_backing_size legacy_backing_size *= units.Gi # Create the legacy backing file if necessary. if legacy_backing_size: if not os.path.exists(legacy_base): with fileutils.remove_path_on_error(legacy_base): libvirt_utils.copy_image(base, legacy_base) image = imgmodel.LocalFileImage(legacy_base, imgmodel.FORMAT_QCOW2) disk.extend(image, legacy_backing_size) if not os.path.exists(self.path): with fileutils.remove_path_on_error(self.path): copy_qcow2_image(base, self.path, size)
def _handle_base_image(self, img_id, base_file): """Handle the checks for a single base image.""" image_in_use = False LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), {'id': img_id, 'base_file': base_file}) if base_file in self.unexplained_images: self.unexplained_images.remove(base_file) if img_id in self.used_images: local, remote, instances = self.used_images[img_id] if local > 0 or remote > 0: image_in_use = True LOG.info(_LI('image %(id)s at (%(base_file)s): ' 'in use: on this node %(local)d local, ' '%(remote)d on other nodes sharing this instance ' 'storage'), {'id': img_id, 'base_file': base_file, 'local': local, 'remote': remote}) self.active_base_files.append(base_file) if not base_file: LOG.warning(_LW('image %(id)s at (%(base_file)s): warning ' '-- an absent base file is in use! ' 'instances: %(instance_list)s'), {'id': img_id, 'base_file': base_file, 'instance_list': ' '.join(instances)}) if base_file: if not image_in_use: LOG.debug('image %(id)s at (%(base_file)s): image is not in ' 'use', {'id': img_id, 'base_file': base_file}) self.removable_base_files.append(base_file) else: LOG.debug('image %(id)s at (%(base_file)s): image is in ' 'use', {'id': img_id, 'base_file': base_file}) if os.path.exists(base_file): libvirt_utils.update_mtime(base_file)
def create_image(self, prepare_template, base, size, *args, **kwargs): filename = self._get_lock_name(base) @utils.synchronized(filename, external=True, lock_path=self.lock_path) def copy_qcow2_image(base, target, size): # TODO(pbrady): Consider copying the cow image here # with preallocation=metadata set for performance reasons. # This would be keyed on a 'preallocate_images' setting. libvirt_utils.create_cow_image(base, target) if size: image = imgmodel.LocalFileImage(target, imgmodel.FORMAT_QCOW2) disk.extend(image, size) # Download the unmodified base image unless we already have a copy. if not os.path.exists(base): prepare_template(target=base, max_size=size, *args, **kwargs) # NOTE(ankit): Update the mtime of the base file so the image # cache manager knows it is in use. libvirt_utils.update_mtime(base) self.verify_base_size(base, size) legacy_backing_size = None legacy_base = base # Determine whether an existing qcow2 disk uses a legacy backing by # actually looking at the image itself and parsing the output of the # backing file it expects to be using. if os.path.exists(self.path): backing_path = libvirt_utils.get_disk_backing_file(self.path) if backing_path is not None: backing_file = os.path.basename(backing_path) backing_parts = backing_file.rpartition('_') if backing_file != backing_parts[-1] and \ backing_parts[-1].isdigit(): legacy_backing_size = int(backing_parts[-1]) legacy_base += '_%d' % legacy_backing_size legacy_backing_size *= units.Gi # Create the legacy backing file if necessary. if legacy_backing_size: if not os.path.exists(legacy_base): with fileutils.remove_path_on_error(legacy_base): libvirt_utils.copy_image(base, legacy_base) image = imgmodel.LocalFileImage(legacy_base, imgmodel.FORMAT_QCOW2) disk.extend(image, legacy_backing_size) if not os.path.exists(self.path): with fileutils.remove_path_on_error(self.path): copy_qcow2_image(base, self.path, size)
def _mark_in_use(self, img_id, base_file): """Mark a single base image as in use.""" LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), {'id': img_id, 'base_file': base_file}) if base_file in self.unexplained_images: self.unexplained_images.remove(base_file) self.active_base_files.append(base_file) LOG.debug('image %(id)s at (%(base_file)s): image is in use', {'id': img_id, 'base_file': base_file}) libvirt_utils.update_mtime(base_file)
def _age_and_verify_swap_images(self, context, base_dir): LOG.debug('Verify swap images') for ent in self.back_swap_images: base_file = os.path.join(base_dir, ent) if ent in self.used_swap_images and os.path.exists(base_file): libvirt_utils.update_mtime(base_file) elif self.remove_unused_base_images: self._remove_swap_file(base_file) error_images = self.used_swap_images - self.back_swap_images for error_image in error_images: LOG.warn(_LW('%s swap image was used by instance' ' but no back files existing!'), error_image)
def _age_and_verify_swap_images(self, context, base_dir): LOG.debug('Verify swap images') for ent in self.back_swap_images: base_file = os.path.join(base_dir, ent) if ent in self.used_swap_images and os.path.exists(base_file): libvirt_utils.update_mtime(base_file) elif self.remove_unused_base_images: self._remove_swap_file(base_file) error_images = self.used_swap_images - self.back_swap_images for error_image in error_images: LOG.warning(_LW('%s swap image was used by instance' ' but no back files existing!'), error_image)
def _mark_in_use(self, img_id, base_file): """Mark a single base image as in use.""" LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), { 'id': img_id, 'base_file': base_file }) if base_file in self.unexplained_images: self.unexplained_images.remove(base_file) self.active_base_files.append(base_file) LOG.debug('image %(id)s at (%(base_file)s): image is in use', { 'id': img_id, 'base_file': base_file }) libvirt_utils.update_mtime(base_file)
def _handle_base_image(self, img_id, base_file): """Handle the checks for a single base image.""" image_bad = False image_in_use = False LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), {'id': img_id, 'base_file': base_file}) if base_file in self.unexplained_images: self.unexplained_images.remove(base_file) if (base_file and os.path.exists(base_file) and os.path.isfile(base_file)): # _verify_checksum returns True if the checksum is ok, and None if # there is no checksum file checksum_result = self._verify_checksum(img_id, base_file) if checksum_result is not None: image_bad = not checksum_result # Give other threads a chance to run time.sleep(0) if img_id in self.used_images: local, remote, instances = self.used_images[img_id] if local > 0 or remote > 0: image_in_use = True LOG.info(_LI('image %(id)s at (%(base_file)s): ' 'in use: on this node %(local)d local, ' '%(remote)d on other nodes sharing this instance ' 'storage'), {'id': img_id, 'base_file': base_file, 'local': local, 'remote': remote}) self.active_base_files.append(base_file) if not base_file: LOG.warn(_LW('image %(id)s at (%(base_file)s): warning ' '-- an absent base file is in use! ' 'instances: %(instance_list)s'), {'id': img_id, 'base_file': base_file, 'instance_list': ' '.join(instances)}) if image_bad: self.corrupt_base_files.append(base_file) if base_file: if not image_in_use: LOG.debug('image %(id)s at (%(base_file)s): image is not in ' 'use', {'id': img_id, 'base_file': base_file}) self.removable_base_files.append(base_file) else: LOG.debug('image %(id)s at (%(base_file)s): image is in ' 'use', {'id': img_id, 'base_file': base_file}) if os.path.exists(base_file): libvirt_utils.update_mtime(base_file)
def _handle_base_image(self, img_id, base_file): """Handle the checks for a single base image.""" image_bad = False image_in_use = False LOG.info(_LI('image %(id)s at (%(base_file)s): checking'), { 'id': img_id, 'base_file': base_file }) if base_file in self.unexplained_images: self.unexplained_images.remove(base_file) if (base_file and os.path.exists(base_file) and os.path.isfile(base_file)): # _verify_checksum returns True if the checksum is ok, and None if # there is no checksum file checksum_result = self._verify_checksum(img_id, base_file) if checksum_result is not None: image_bad = not checksum_result # Give other threads a chance to run time.sleep(0) if img_id in self.used_images: local, remote, instances = self.used_images[img_id] if local > 0 or remote > 0: image_in_use = True LOG.info( _LI('image %(id)s at (%(base_file)s): ' 'in use: on this node %(local)d local, ' '%(remote)d on other nodes sharing this instance ' 'storage'), { 'id': img_id, 'base_file': base_file, 'local': local, 'remote': remote }) self.active_base_files.append(base_file) if not base_file: LOG.warning( _LW('image %(id)s at (%(base_file)s): warning ' '-- an absent base file is in use! ' 'instances: %(instance_list)s'), { 'id': img_id, 'base_file': base_file, 'instance_list': ' '.join(instances) }) if image_bad: self.corrupt_base_files.append(base_file) if base_file: if not image_in_use: LOG.debug( 'image %(id)s at (%(base_file)s): image is not in ' 'use', { 'id': img_id, 'base_file': base_file }) self.removable_base_files.append(base_file) else: LOG.debug( 'image %(id)s at (%(base_file)s): image is in ' 'use', { 'id': img_id, 'base_file': base_file }) if os.path.exists(base_file): libvirt_utils.update_mtime(base_file)