def _create_metadata_artifact(self, size, vol_format, prealloc, disk_type, desc, parent): if self._oop.fileUtils.pathExists(self.meta_path): raise se.VolumeAlreadyExists("metadata exists: %r" % self.meta_path) if self._oop.fileUtils.pathExists(self.meta_volatile_path): raise se.DomainHasGarbage("metadata artifact exists: %r" % self.meta_volatile_path) parent_vol_id = parent.vol_id if parent else sc.BLANK_UUID # Create the metadata artifact. The metadata file is created with a # special extension to prevent these artifacts from being recognized as # a volume until FileVolumeArtifacts.commit() is called. meta = VolumeMetadata( self.sd_manifest.sdUUID, self.img_id, parent_vol_id, size / sc.BLOCK_SIZE, # Size is stored as number of blocks sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(sc.LEAF_VOL), disk_type, desc, sc.LEGAL_VOL) self._oop.writeFile(self.meta_volatile_path, meta.storage_format())
def make_file_volume(sd_manifest, size, imguuid, voluuid, parent_vol_id=sc.BLANK_UUID, vol_format=sc.RAW_FORMAT, vol_type=sc.LEAF_VOL, prealloc=sc.SPARSE_VOL, disk_type=image.UNKNOWN_DISK_TYPE, desc='fake volume'): volpath = os.path.join(sd_manifest.domaindir, "images", imguuid, voluuid) mdfiles = [volpath + '.meta', volpath + '.lease'] make_file(volpath, size) for mdfile in mdfiles: make_file(mdfile) size_blk = size / sc.BLOCK_SIZE vol_class = sd_manifest.getVolumeClass() vol_class.newMetadata( (volpath,), sd_manifest.sdUUID, imguuid, parent_vol_id, size_blk, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(vol_type), disk_type, desc, sc.LEGAL_VOL)
def _create_metadata(self, meta_slot, size, vol_format, prealloc, disk_type, desc, parent): # When the volume format is RAW the real volume capacity is the device # size. The device size may have been rounded up if 'size' is not # divisible by the domain's extent size. if vol_format == sc.RAW_FORMAT: size = int(self.sd_manifest.getVSize(self.img_id, self.vol_id)) # We use the BlockVolumeManifest API here because we create the # metadata in the standard way. We cannot do this for file volumes # because the metadata needs to be written to a specially named file. meta_id = (self.sd_manifest.sdUUID, meta_slot) parent_vol_id = parent.vol_id if parent else sc.BLANK_UUID size_blk = size / sc.BLOCK_SIZE self.vol_class.newMetadata( meta_id, self.sd_manifest.sdUUID, self.img_id, parent_vol_id, size_blk, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(sc.LEAF_VOL), disk_type, desc, sc.LEGAL_VOL)
def make_block_volume(lvm, sd_manifest, size, imguuid, voluuid, parent_vol_id=sc.BLANK_UUID, vol_format=sc.RAW_FORMAT, vol_type=sc.LEAF_VOL, prealloc=sc.PREALLOCATED_VOL, disk_type=sc.DATA_DISKTYPE, desc='fake volume', qcow2_compat='0.10'): sduuid = sd_manifest.sdUUID imagedir = sd_manifest.getImageDir(imguuid) if not os.path.exists(imagedir): os.makedirs(imagedir) size_blk = (size + sc.BLOCK_SIZE - 1) // sc.BLOCK_SIZE lv_size = sd_manifest.getVolumeClass().calculate_volume_alloc_size( prealloc, size_blk, None) lvm.createLV(sduuid, voluuid, lv_size) # LVM may create the volume with a larger size due to extent granularity lv_size_blk = int(lvm.getLV(sduuid, voluuid).size) // sc.BLOCK_SIZE if lv_size_blk > size_blk: size_blk = lv_size_blk if vol_format == sc.COW_FORMAT: volpath = lvm.lvPath(sduuid, voluuid) backing = parent_vol_id if parent_vol_id != sc.BLANK_UUID else None # Write qcow2 image to the fake block device - truncating the file. op = qemuimg.create( volpath, size=size, format=qemuimg.FORMAT.QCOW2, qcow2Compat=qcow2_compat, backing=backing) op.run() # Truncate fake block device back ot the proper size. with open(volpath, "r+") as f: f.truncate(int(lvm.getLV(sduuid, voluuid).size)) with sd_manifest.acquireVolumeMetadataSlot(voluuid) as slot: lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_MD, slot)) lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_PARENT, parent_vol_id)) lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_IMAGE, imguuid)) vol_class = sd_manifest.getVolumeClass() vol_class.newMetadata( (sduuid, slot), sduuid, imguuid, parent_vol_id, size_blk, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(vol_type), disk_type, desc, sc.LEGAL_VOL)
def test_set_type(self, env_type): with self.make_env(env_type) as env: leaf_vol = env.chain[0] generation = leaf_vol.getMetaParam(sc.GENERATION) job = update_volume.Job(make_uuid(), 0, make_endpoint_from_volume(leaf_vol), dict(type=sc.type2name(sc.SHARED_VOL))) job.run() self.assertEqual(jobs.STATUS.DONE, job.status) self.assertEqual(sc.type2name(sc.SHARED_VOL), leaf_vol.getMetaParam(sc.VOLTYPE)) self.assertEqual(generation + 1, leaf_vol.getMetaParam(sc.GENERATION))
def test_set_type_leaf_with_parent(self, env_type): with self.make_env(env_type, chain_length=2) as env: top_vol = env.chain[1] generation = top_vol.getMetaParam(sc.GENERATION) job = update_volume.Job(make_uuid(), 0, make_endpoint_from_volume(top_vol), dict(type=sc.type2name(sc.SHARED_VOL))) job.run() self.assertEqual(job.status, jobs.STATUS.FAILED) self.assertEqual(type(job.error), se.InvalidVolumeUpdate) self.assertEqual(sc.type2name(sc.LEAF_VOL), top_vol.getMetaParam(sc.VOLTYPE)) self.assertEqual(generation, top_vol.getMetaParam(sc.GENERATION))
def make_init_params(**kwargs): res = dict( domain=str(uuid.uuid4()), image=str(uuid.uuid4()), puuid=str(uuid.uuid4()), size=1024 * MB, format=sc.type2name(sc.RAW_FORMAT), type=sc.type2name(sc.SPARSE_VOL), voltype=sc.type2name(sc.LEAF_VOL), disktype=image.SYSTEM_DISK_TYPE, description="", legality=sc.LEGAL_VOL) res.update(kwargs) return res
def make_init_params(**kwargs): res = dict( domain=make_uuid(), image=make_uuid(), puuid=make_uuid(), capacity=1024 * MB, format=sc.type2name(sc.RAW_FORMAT), type=sc.type2name(sc.SPARSE_VOL), voltype=sc.type2name(sc.LEAF_VOL), disktype=image.SYSTEM_DISK_TYPE, description="", legality=sc.LEGAL_VOL, generation=sc.DEFAULT_GENERATION) res.update(kwargs) return res
def test_volume_type(self, vol_type): with fake_block_env() as env: img_id = make_uuid() vol_id = make_uuid() make_block_volume(env.lvm, env.sd_manifest, 0, img_id, vol_id, vol_type=vol_type) vol = env.sd_manifest.produceVolume(img_id, vol_id) self.assertEqual(vol.getVolType(), sc.type2name(vol_type))
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ if initialSize: cls.log.error("initialSize is not supported for file-based " "volumes") raise se.InvalidParameterException("initial size", initialSize) sizeBytes = size * BLOCK_SIZE truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0 try: oop.getProcessPool(dom.sdUUID).truncateFile( volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == sc.PREALLOCATED_VOL: try: operation = fallocate.allocate(volPath, sizeBytes) with vars.task.abort_callback(operation.abort): with utils.stopwatch("Preallocating volume %s" % volPath): operation.run() except exception.ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info("Request to create %s volume %s with size = %s " "sectors", sc.type2name(volFormat), volPath, size) if volFormat == sc.COW_FORMAT: qemuimg.create(volPath, size=sizeBytes, format=sc.fmt2str(volFormat), qcow2Compat=dom.qcow2_compat()) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(volPath, volFormat) # Forcing the volume permissions in case one of the tools we use # (dd, qemu-img, etc.) will mistakenly change the file permissiosn. dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS) return (volPath,)
def make_block_volume(lvm, sd_manifest, size, imguuid, voluuid, parent_vol_id=sc.BLANK_UUID, vol_format=sc.RAW_FORMAT, vol_type=sc.LEAF_VOL, prealloc=sc.PREALLOCATED_VOL, disk_type=image.UNKNOWN_DISK_TYPE, desc='fake volume'): sduuid = sd_manifest.sdUUID image_manifest = image.ImageManifest(sd_manifest.getRepoPath()) imagedir = image_manifest.getImageDir(sduuid, imguuid) if not os.path.exists(imagedir): os.makedirs(imagedir) size_blk = (size + sc.BLOCK_SIZE - 1) / sc.BLOCK_SIZE lv_size = sd_manifest.getVolumeClass().calculate_volume_alloc_size( prealloc, size_blk, None) lvm.createLV(sduuid, voluuid, lv_size) # LVM may create the volume with a larger size due to extent granularity lv_size_blk = int(lvm.getLV(sduuid, voluuid).size) / sc.BLOCK_SIZE if lv_size_blk > size_blk: size_blk = lv_size_blk with sd_manifest.acquireVolumeMetadataSlot( voluuid, sc.VOLUME_MDNUMBLKS) as slot: lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_MD, slot)) lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_PARENT, parent_vol_id)) lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_IMAGE, imguuid)) vol_class = sd_manifest.getVolumeClass() vol_class.newMetadata( (sduuid, slot), sduuid, imguuid, parent_vol_id, size_blk, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(vol_type), disk_type, desc, sc.LEGAL_VOL)
def make_file_volume(sd_manifest, size, imguuid, voluuid, parent_vol_id=sc.BLANK_UUID, vol_format=sc.RAW_FORMAT, vol_type=sc.LEAF_VOL, prealloc=sc.SPARSE_VOL, disk_type=sc.DATA_DISKTYPE, desc='fake volume', qcow2_compat='0.10'): volpath = os.path.join(sd_manifest.domaindir, "images", imguuid, voluuid) # Create needed path components. make_file(volpath, size) # Create qcow2 file if needed. if vol_format == sc.COW_FORMAT: backing = parent_vol_id if parent_vol_id != sc.BLANK_UUID else None op = qemuimg.create( volpath, size=size, format=qemuimg.FORMAT.QCOW2, qcow2Compat=qcow2_compat, backing=backing) op.run() # Create meta files. mdfiles = [volpath + '.meta', volpath + '.lease'] for mdfile in mdfiles: make_file(mdfile) size_blk = size // sc.BLOCK_SIZE vol_class = sd_manifest.getVolumeClass() vol_class.newMetadata( (volpath,), sd_manifest.sdUUID, imguuid, parent_vol_id, size_blk, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(vol_type), disk_type, desc, sc.LEGAL_VOL)
def test_set_type_internal(self, env_type): with self.make_env(env_type, chain_length=1) as env: internal_vol = env.chain[0] generation = internal_vol.getMetaParam(sc.GENERATION) internal_vol.setInternal() job = update_volume.Job(make_uuid(), 0, make_endpoint_from_volume(internal_vol), dict(type=sc.type2name(sc.SHARED_VOL))) job.run() self.assertEqual(job.status, jobs.STATUS.FAILED) self.assertEqual(type(job.error), se.InvalidVolumeUpdate) self.assertEqual(generation, internal_vol.getMetaParam(sc.GENERATION))
def validateCreateVolumeParams(self, volFormat, srcVolUUID, preallocate=None): """ Validate create volume parameters """ if volFormat not in sc.VOL_FORMAT: raise se.IncorrectFormat(volFormat) # Volumes with a parent must be cow if srcVolUUID != sc.BLANK_UUID and volFormat != sc.COW_FORMAT: raise se.IncorrectFormat(sc.type2name(volFormat)) if preallocate is not None and preallocate not in sc.VOL_TYPE: raise se.IncorrectType(preallocate)
def make_block_volume(lvm, sd_manifest, size, imguuid, voluuid, parent_vol_id=sc.BLANK_UUID, vol_format=sc.RAW_FORMAT, prealloc=sc.PREALLOCATED_VOL, disk_type=image.UNKNOWN_DISK_TYPE, desc='fake volume'): sduuid = sd_manifest.sdUUID image_manifest = image.ImageManifest(sd_manifest.getRepoPath()) imagedir = image_manifest.getImageDir(sduuid, imguuid) if not os.path.exists(imagedir): os.makedirs(imagedir) size_mb = utils.round(size, MB) / MB lvm.createLV(sduuid, voluuid, size_mb) lv_size = int(lvm.getLV(sduuid, voluuid).size) lv_size_blk = lv_size / sc.BLOCK_SIZE with sd_manifest.acquireVolumeMetadataSlot( voluuid, sc.VOLUME_MDNUMBLKS) as slot: lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_MD, slot)) lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_PARENT, sc.BLANK_UUID)) lvm.addtag(sduuid, voluuid, "%s%s" % (sc.TAG_PREFIX_IMAGE, imguuid)) vol_class = sd_manifest.getVolumeClass() vol_class.newMetadata( (sduuid, slot), sduuid, imguuid, parent_vol_id, lv_size_blk, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(sc.LEAF_VOL), disk_type, desc, sc.LEGAL_VOL)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ lvSize = cls.calculate_volume_alloc_size(preallocate, size, initialSize) lvm.createLV(dom.sdUUID, volUUID, lvSize, activate=True, initialTags=(sc.TAG_VOL_UNINIT,)) fileutils.rm_file(volPath) lvPath = lvm.lvPath(dom.sdUUID, volUUID) cls.log.info("Creating volume symlink from %r to %r", lvPath, volPath) os.symlink(lvPath, volPath) if not volParent: cls.log.info("Request to create %s volume %s with size = %s " "blocks", sc.type2name(volFormat), volPath, size) if volFormat == sc.COW_FORMAT: operation = qemuimg.create(volPath, size=size * BLOCK_SIZE, format=sc.fmt2str(volFormat), qcow2Compat=dom.qcow2_compat()) operation.run() else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s " "with size %s (blocks)", imgUUID, volUUID, srcImgUUID, srcVolUUID, size) volParent.clone(volPath, volFormat, size) with dom.acquireVolumeMetadataSlot(volUUID) as slot: mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot), "%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID), "%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)] lvm.changeLVTags(dom.sdUUID, volUUID, delTags=[sc.TAG_VOL_UNINIT], addTags=mdTags) try: lvm.deactivateLVs(dom.sdUUID, [volUUID]) except se.CannotDeactivateLogicalVolume: cls.log.warn("Cannot deactivate new created volume %s/%s", dom.sdUUID, volUUID, exc_info=True) return (dom.sdUUID, slot)
def test_new_image_create_and_commit(self): with self.fake_env() as env: artifacts = env.sd_manifest.get_volume_artifacts( self.img_id, self.vol_id) size, vol_format, disk_type, desc = BASE_PARAMS[sc.RAW_FORMAT] artifacts.create(size, vol_format, disk_type, desc) artifacts.commit() vol = env.sd_manifest.produceVolume(self.img_id, self.vol_id) self.assertEqual(sc.type2name(sc.LEAF_VOL), vol.getVolType()) self.assertEqual(desc, vol.getDescription()) self.assertEqual(sc.LEGAL_VOL, vol.getLegality()) self.assertEqual(size / sc.BLOCK_SIZE, vol.getSize()) self.assertEqual(size, os.stat(artifacts.volume_path).st_size) self.assertEqual(vol_format, vol.getFormat()) self.assertEqual(str(disk_type), vol.getDiskType())
def validateCreateVolumeParams(cls, volFormat, srcVolUUID, diskType=None, preallocate=None): """ Validate create volume parameters """ if volFormat not in sc.VOL_FORMAT: raise se.IncorrectFormat(volFormat) # Volumes with a parent must be cow if srcVolUUID != sc.BLANK_UUID and volFormat != sc.COW_FORMAT: raise se.IncorrectFormat(sc.type2name(volFormat)) if diskType is not None and diskType not in sc.VOL_DISKTYPE: raise se.InvalidParameterException("DiskType", diskType) if preallocate is not None and preallocate not in sc.VOL_TYPE: raise se.IncorrectType(preallocate)
def validateCreateVolumeParams(cls, volFormat, srcVolUUID, diskType=None, preallocate=None, add_bitmaps=False): """ Validate create volume parameters """ if volFormat not in sc.VOL_FORMAT: raise se.IncorrectFormat(volFormat) # Volumes with a parent must be cow if srcVolUUID != sc.BLANK_UUID and volFormat != sc.COW_FORMAT: raise se.IncorrectFormat(sc.type2name(volFormat)) if diskType is not None and diskType not in sc.VOL_DISKTYPE: raise se.InvalidParameterException("DiskType", diskType) if preallocate is not None and preallocate not in sc.VOL_TYPE: raise se.IncorrectType(preallocate) if add_bitmaps and srcVolUUID == sc.BLANK_UUID: raise se.UnsupportedOperation( "Cannot add bitmaps for volume without parent volume", srcVolUUID=srcVolUUID, add_bitmaps=add_bitmaps)
def setInternal(self): self.setMetaParam(sc.VOLTYPE, sc.type2name(sc.INTERNAL_VOL)) self.voltype = sc.type2name(sc.INTERNAL_VOL) self.setrw(rw=False) return self.voltype
def setFormat(self, volFormat): self.setMetaParam(sc.FORMAT, sc.type2name(volFormat))
def setShared(self): self.setMetaParam(sc.VOLTYPE, sc.type2name(sc.SHARED_VOL)) self.voltype = sc.type2name(sc.SHARED_VOL) self.setrw(rw=False) return self.voltype
def setLeaf(self): self.setMetaParam(sc.VOLTYPE, sc.type2name(sc.LEAF_VOL)) self.voltype = sc.type2name(sc.LEAF_VOL) self.setrw(rw=True) return self.voltype
def setType(self, prealloc): self.setMetaParam(sc.TYPE, sc.type2name(prealloc))
def _validate_type(self): if self.type is not None: if self.type != sc.type2name(sc.SHARED_VOL): raise ValueError("Volume type not supported %s" % self.type)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ lvSize = cls.calculate_volume_alloc_size(preallocate, size, initialSize) lvm.createLV(dom.sdUUID, volUUID, "%s" % lvSize, activate=True, initialTags=(sc.TAG_VOL_UNINIT, )) utils.rmFile(volPath) os.symlink(lvm.lvPath(dom.sdUUID, volUUID), volPath) if not volParent: cls.log.info( "Request to create %s volume %s with size = %s " "sectors", sc.type2name(volFormat), volPath, size) if volFormat == sc.COW_FORMAT: qemuimg.create(volPath, size * BLOCK_SIZE, sc.fmt2str(volFormat)) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(volPath, volFormat) with dom.acquireVolumeMetadataSlot(volUUID, sc.VOLUME_MDNUMBLKS) as slot: mdTags = [ "%s%s" % (sc.TAG_PREFIX_MD, slot), "%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID), "%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID) ] lvm.changeLVTags(dom.sdUUID, volUUID, delTags=[sc.TAG_VOL_UNINIT], addTags=mdTags) try: lvm.deactivateLVs(dom.sdUUID, [volUUID]) except se.CannotDeactivateLogicalVolume: cls.log.warn("Cannot deactivate new created volume %s/%s", dom.sdUUID, volUUID, exc_info=True) return (dom.sdUUID, slot)
def isShared(self): return self.getVolType() == sc.type2name(sc.SHARED_VOL)
def isInternal(self): return self.getVolType() == sc.type2name(sc.INTERNAL_VOL)
def test_volume_life_cycle(monkeypatch, user_domain): # as creation of block storage domain and volume is quite time consuming, # we test several volume operations in one test to speed up the test suite img_uuid = str(uuid.uuid4()) vol_uuid = str(uuid.uuid4()) vol_capacity = 10 * GiB vol_desc = "Test volume" with monkeypatch.context() as mc: mc.setattr(time, "time", lambda: 1550522547) user_domain.createVolume(imgUUID=img_uuid, capacity=vol_capacity, volFormat=sc.COW_FORMAT, preallocate=sc.SPARSE_VOL, diskType=sc.DATA_DISKTYPE, volUUID=vol_uuid, desc=vol_desc, srcImgUUID=sc.BLANK_UUID, srcVolUUID=sc.BLANK_UUID) # test create volume vol = user_domain.produceVolume(img_uuid, vol_uuid) actual = vol.getInfo() assert int(actual["capacity"]) == vol_capacity assert int(actual["ctime"]) == 1550522547 assert actual["description"] == vol_desc assert actual["disktype"] == "DATA" assert actual["domain"] == user_domain.sdUUID assert actual["format"] == sc.type2name(sc.COW_FORMAT) assert actual["parent"] == sc.BLANK_UUID assert actual["status"] == sc.VOL_STATUS_OK assert actual["type"] == "SPARSE" assert actual["voltype"] == "LEAF" assert actual["uuid"] == vol_uuid vol_path = vol.getVolumePath() qcow2_info = qemuimg.info(vol_path) assert qcow2_info["actual-size"] < vol_capacity assert qcow2_info["virtual-size"] == vol_capacity size = user_domain.getVolumeSize(img_uuid, vol_uuid) assert size.apparentsize == os.path.getsize(vol_path) assert size.truesize == qcow2_info["actual-size"] # test volume prepare, teardown does nothing in case of file volume vol.prepare() permissions = stat.S_IMODE(os.stat(vol_path).st_mode) assert permissions == sc.FILE_VOLUME_PERMISSIONS # verify we can really write and read to an image qemuio.write_pattern(vol_path, "qcow2") qemuio.verify_pattern(vol_path, "qcow2") # test deleting of the volume - check volume and metadata files are # deleted once the volume is deleted. Lock files is not checked as it's # not created in case of file volume which uses LocalLock vol_path = vol.getVolumePath() meta_path = vol._manifest.metaVolumePath(vol_path) assert os.path.isfile(vol_path) assert os.path.isfile(meta_path) vol.delete(postZero=False, force=False, discard=False) assert not os.path.isfile(vol_path) assert not os.path.isfile(meta_path)
def create(cls, repoPath, sdUUID, imgUUID, size, volFormat, preallocate, diskType, volUUID, desc, srcImgUUID, srcVolUUID, initialSize=None): """ Create a new volume with given size or snapshot 'size' - in sectors 'volFormat' - volume format COW / RAW 'preallocate' - Preallocate / Sparse 'diskType' - enum (API.Image.DiskTypes) 'srcImgUUID' - source image UUID 'srcVolUUID' - source volume UUID 'initialSize' - initial volume size in sectors, in case of thin provisioning """ dom = sdCache.produce(sdUUID) dom.validateCreateVolumeParams(volFormat, srcVolUUID, preallocate=preallocate) imgPath = image.Image(repoPath).create(sdUUID, imgUUID) volPath = os.path.join(imgPath, volUUID) volParent = None volType = sc.type2name(sc.LEAF_VOL) # Get the specific class name and class module to be used in the # Recovery tasks. clsModule, clsName = cls._getModuleAndClass() try: if srcVolUUID != sc.BLANK_UUID: # When the srcImgUUID isn't specified we assume it's the same # as the imgUUID if srcImgUUID == sc.BLANK_UUID: srcImgUUID = imgUUID volParent = cls(repoPath, sdUUID, srcImgUUID, srcVolUUID) if not volParent.isLegal(): raise se.createIllegalVolumeSnapshotError( volParent.volUUID) if imgUUID != srcImgUUID: volParent.share(imgPath) volParent = cls(repoPath, sdUUID, imgUUID, srcVolUUID) # Override the size with the size of the parent size = volParent.getSize() except se.StorageException: cls.log.error("Unexpected error", exc_info=True) raise except Exception as e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumeCannotGetParent( "Couldn't get parent %s for volume %s: %s" % (srcVolUUID, volUUID, e)) try: cls.log.info("Creating volume %s", volUUID) # Rollback sentinel to mark the start of the task vars.task.pushRecovery( task.Recovery(task.ROLLBACK_SENTINEL, clsModule, clsName, "startCreateVolumeRollback", [sdUUID, imgUUID, volUUID]) ) # Create volume rollback vars.task.pushRecovery( task.Recovery("Halfbaked volume rollback", clsModule, clsName, "halfbakedVolumeRollback", [sdUUID, volUUID, volPath]) ) # Specific volume creation (block, file, etc...) try: metaId = cls._create(dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=initialSize) except (se.VolumeAlreadyExists, se.CannotCreateLogicalVolume, se.VolumeCreationError, se.InvalidParameterException) as e: cls.log.error("Failed to create volume %s: %s", volPath, e) vars.task.popRecovery() raise # When the volume format is raw what the guest sees is the apparent # size of the file/device therefore if the requested size doesn't # match the apparent size (eg: physical extent granularity in LVM) # we need to update the size value so that the metadata reflects # the correct state. if volFormat == sc.RAW_FORMAT: apparentSize = int(dom.getVSize(imgUUID, volUUID) / sc.BLOCK_SIZE) if apparentSize < size: cls.log.error("The volume %s apparent size %s is smaller " "than the requested size %s", volUUID, apparentSize, size) raise se.VolumeCreationError() if apparentSize > size: cls.log.info("The requested size for volume %s doesn't " "match the granularity on domain %s, " "updating the volume size from %s to %s", volUUID, sdUUID, size, apparentSize) size = apparentSize vars.task.pushRecovery( task.Recovery("Create volume metadata rollback", clsModule, clsName, "createVolumeMetadataRollback", map(str, metaId)) ) cls.newMetadata(metaId, sdUUID, imgUUID, srcVolUUID, size, sc.type2name(volFormat), sc.type2name(preallocate), volType, diskType, desc, sc.LEGAL_VOL) if dom.hasVolumeLeases(): cls.newVolumeLease(metaId, sdUUID, volUUID) except se.StorageException: cls.log.error("Unexpected error", exc_info=True) raise except Exception as e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumeCreationError("Volume creation %s failed: %s" % (volUUID, e)) # Remove the rollback for the halfbaked volume vars.task.replaceRecoveries( task.Recovery("Create volume rollback", clsModule, clsName, "createVolumeRollback", [repoPath, sdUUID, imgUUID, volUUID, imgPath]) ) return volUUID
def _create(cls, dom, imgUUID, volUUID, capacity, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initial_size=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ lv_size = cls.calculate_volume_alloc_size(preallocate, volFormat, capacity, initial_size) lv_size_mb = utils.round(lv_size, MiB) // MiB lvm.createLV(dom.sdUUID, volUUID, lv_size_mb, activate=True, initialTags=(sc.TAG_VOL_UNINIT, )) fileutils.rm_file(volPath) lvPath = lvm.lvPath(dom.sdUUID, volUUID) cls.log.info("Creating volume symlink from %r to %r", lvPath, volPath) os.symlink(lvPath, volPath) if not volParent: cls.log.info("Request to create %s volume %s with capacity = %s", sc.type2name(volFormat), volPath, capacity) if volFormat == sc.COW_FORMAT: operation = qemuimg.create(volPath, size=capacity, format=sc.fmt2str(volFormat), qcow2Compat=dom.qcow2_compat()) operation.run() else: # Create hardlink to template and its meta file cls.log.info( "Request to create snapshot %s/%s of volume %s/%s " "with capacity %s", imgUUID, volUUID, srcImgUUID, srcVolUUID, capacity) volParent.clone(volPath, volFormat, capacity) with dom.acquireVolumeMetadataSlot(volUUID) as slot: mdTags = [ "%s%s" % (sc.TAG_PREFIX_MD, slot), "%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID), "%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID) ] lvm.changeLVsTags(dom.sdUUID, (volUUID, ), delTags=[sc.TAG_VOL_UNINIT], addTags=mdTags) try: lvm.deactivateLVs(dom.sdUUID, [volUUID]) except se.CannotDeactivateLogicalVolume: cls.log.warn("Cannot deactivate new created volume %s/%s", dom.sdUUID, volUUID, exc_info=True) return (dom.sdUUID, slot)
def make_block_volume(lvm, sd_manifest, size, imguuid, voluuid, parent_vol_id=sc.BLANK_UUID, vol_format=sc.RAW_FORMAT, vol_type=sc.LEAF_VOL, prealloc=sc.PREALLOCATED_VOL, disk_type=sc.DATA_DISKTYPE, desc='fake volume', qcow2_compat='0.10'): sduuid = sd_manifest.sdUUID imagedir = sd_manifest.getImageDir(imguuid) if not os.path.exists(imagedir): os.makedirs(imagedir) lv_size = sd_manifest.getVolumeClass().calculate_volume_alloc_size( prealloc, vol_format, size, None) lv_size_mb = (utils.round(lv_size, constants.MEGAB) // constants.MEGAB) lvm.createLV(sduuid, voluuid, lv_size_mb) # LVM may create the volume with a larger size due to extent granularity if vol_format == sc.RAW_FORMAT: lv_size = int(lvm.getLV(sduuid, voluuid).size) if lv_size > size: size = lv_size if vol_format == sc.COW_FORMAT: volpath = lvm.lvPath(sduuid, voluuid) backing = parent_vol_id if parent_vol_id != sc.BLANK_UUID else None # Write qcow2 image to the fake block device - truncating the file. op = qemuimg.create( volpath, size=size, format=qemuimg.FORMAT.QCOW2, qcow2Compat=qcow2_compat, backing=backing) op.run() # Truncate fake block device back ot the proper size. with open(volpath, "r+") as f: f.truncate(int(lvm.getLV(sduuid, voluuid).size)) with sd_manifest.acquireVolumeMetadataSlot(voluuid) as slot: add_tags = [ "%s%s" % (sc.TAG_PREFIX_MD, slot), "%s%s" % (sc.TAG_PREFIX_PARENT, parent_vol_id), "%s%s" % (sc.TAG_PREFIX_IMAGE, imguuid), ] lvm.changeLVsTags(sduuid, (voluuid,), addTags=add_tags) vol_class = sd_manifest.getVolumeClass() vol_class.newMetadata( (sduuid, slot), sduuid, imguuid, parent_vol_id, size, sc.type2name(vol_format), sc.type2name(prealloc), sc.type2name(vol_type), disk_type, desc, sc.LEGAL_VOL)
def isLeaf(self): return self.getVolType() == sc.type2name(sc.LEAF_VOL)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ if initialSize: cls.log.error("initialSize is not supported for file-based " "volumes") raise se.InvalidParameterException("initial size", initialSize) sizeBytes = size * BLOCK_SIZE truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0 try: oop.getProcessPool(dom.sdUUID).truncateFile( volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == sc.PREALLOCATED_VOL: try: operation = fallocate.allocate(volPath, sizeBytes) with vars.task.abort_callback(operation.abort): with utils.stopwatch("Preallocating volume %s" % volPath): operation.run() except exception.ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info( "Request to create %s volume %s with size = %s " "blocks", sc.type2name(volFormat), volPath, size) if volFormat == sc.COW_FORMAT: operation = qemuimg.create(volPath, size=sizeBytes, format=sc.fmt2str(volFormat), qcow2Compat=dom.qcow2_compat()) operation.run() else: # Create hardlink to template and its meta file cls.log.info( "Request to create snapshot %s/%s of volume %s/%s " "with size %s (blocks)", imgUUID, volUUID, srcImgUUID, srcVolUUID, size) volParent.clone(volPath, volFormat, size) # Forcing the volume permissions in case one of the tools we use # (dd, qemu-img, etc.) will mistakenly change the file permissiosn. cls.log.info("Changing volume %r permission to %04o", volPath, sc.FILE_VOLUME_PERMISSIONS) dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS) return (volPath, )
def copyCollapsed(self, sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID, dstVolUUID, descr, dstSdUUID, volType, volFormat, preallocate, postZero, force, discard): """ Create new template/volume from VM. Do it by collapse and copy the whole chain (baseVolUUID->srcVolUUID) """ self.log.info( "sdUUID=%s vmUUID=%s srcImgUUID=%s srcVolUUID=%s " "dstImgUUID=%s dstVolUUID=%s dstSdUUID=%s volType=%s " "volFormat=%s preallocate=%s force=%s postZero=%s " "discard=%s", sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID, dstVolUUID, dstSdUUID, volType, sc.type2name(volFormat), sc.type2name(preallocate), str(force), str(postZero), discard) try: srcVol = dstVol = None # Find out dest sdUUID if dstSdUUID == sd.BLANK_UUID: dstSdUUID = sdUUID volclass = sdCache.produce(sdUUID).getVolumeClass() destDom = sdCache.produce(dstSdUUID) # find src volume try: srcVol = volclass(self.repoPath, sdUUID, srcImgUUID, srcVolUUID) except se.StorageException: raise except Exception as e: self.log.error(e, exc_info=True) raise se.SourceImageActionError(srcImgUUID, sdUUID, str(e)) # Create dst volume try: # Before reading source volume parameters from volume metadata, # prepare the volume. This ensure that the volume capacity will # match the actual virtual size, see # https://bugzilla.redhat.com/1700623. srcVol.prepare(rw=False) volParams = srcVol.getVolumeParams() if volFormat in [sc.COW_FORMAT, sc.RAW_FORMAT]: dstVolFormat = volFormat else: dstVolFormat = volParams['volFormat'] # TODO: This is needed only when copying to qcow2-thin volume # on block storage. Move into calculate_initial_size. dst_vol_allocation = self.calculate_vol_alloc( sdUUID, volParams, dstSdUUID, dstVolFormat) # Find out dest volume parameters if preallocate in [sc.PREALLOCATED_VOL, sc.SPARSE_VOL]: volParams['prealloc'] = preallocate initial_size = self.calculate_initial_size( destDom.supportsSparseness, dstVolFormat, volParams['prealloc'], dst_vol_allocation) self.log.info( "Copy source %s:%s:%s to destination %s:%s:%s " "capacity=%s, initial size=%s", sdUUID, srcImgUUID, srcVolUUID, dstSdUUID, dstImgUUID, dstVolUUID, volParams['capacity'], initial_size) # If image already exists check whether it illegal/fake, # overwrite it if not self.isLegal(dstSdUUID, dstImgUUID): force = True # We must first remove the previous instance of image (if # exists) in destination domain, if we got the overwrite # command if force: self.log.info( "delete image %s on domain %s before " "overwriting", dstImgUUID, dstSdUUID) _deleteImage(destDom, dstImgUUID, postZero, discard) destDom.createVolume(imgUUID=dstImgUUID, capacity=volParams['capacity'], volFormat=dstVolFormat, preallocate=volParams['prealloc'], diskType=volParams['disktype'], volUUID=dstVolUUID, desc=descr, srcImgUUID=sc.BLANK_UUID, srcVolUUID=sc.BLANK_UUID, initial_size=initial_size) dstVol = sdCache.produce(dstSdUUID).produceVolume( imgUUID=dstImgUUID, volUUID=dstVolUUID) except se.StorageException: self.log.error("Unexpected error", exc_info=True) raise except Exception as e: self.log.error("Unexpected error", exc_info=True) raise se.CopyImageError("Destination volume %s error: %s" % (dstVolUUID, str(e))) try: # Start the actual copy image procedure dstVol.prepare(rw=True, setrw=True) if (destDom.supportsSparseness and dstVol.getType() == sc.PREALLOCATED_VOL): preallocation = qemuimg.PREALLOCATION.FALLOC else: preallocation = None try: operation = qemuimg.convert( volParams['path'], dstVol.getVolumePath(), srcFormat=sc.fmt2str(volParams['volFormat']), dstFormat=sc.fmt2str(dstVolFormat), dstQcow2Compat=destDom.qcow2_compat(), preallocation=preallocation, unordered_writes=destDom.recommends_unordered_writes( dstVolFormat), create=not destDom.is_block(), ) with utils.stopwatch("Copy volume %s" % srcVol.volUUID): self._run_qemuimg_operation(operation) except ActionStopped: raise except cmdutils.Error as e: self.log.exception('conversion failure for volume %s', srcVol.volUUID) raise se.CopyImageError(str(e)) # Mark volume as SHARED if volType == sc.SHARED_VOL: dstVol.setShared() dstVol.setLegality(sc.LEGAL_VOL) if force: # Now we should re-link all deleted hardlinks, if exists destDom.templateRelink(dstImgUUID, dstVolUUID) except se.StorageException: self.log.error("Unexpected error", exc_info=True) raise except Exception as e: self.log.error("Unexpected error", exc_info=True) raise se.CopyImageError("src image=%s, dst image=%s: msg=%s" % (srcImgUUID, dstImgUUID, str(e))) self.log.info("Finished copying %s:%s -> %s:%s", sdUUID, srcVolUUID, dstSdUUID, dstVolUUID) # TODO: handle return status return dstVolUUID finally: self.__cleanupCopy(srcVol=srcVol, dstVol=dstVol)
def create(cls, repoPath, sdUUID, imgUUID, size, volFormat, preallocate, diskType, volUUID, desc, srcImgUUID, srcVolUUID, initialSize=None): """ Create a new volume with given size or snapshot 'size' - in sectors 'volFormat' - volume format COW / RAW 'preallocate' - Preallocate / Sparse 'diskType' - enum (API.Image.DiskTypes) 'srcImgUUID' - source image UUID 'srcVolUUID' - source volume UUID 'initialSize' - initial volume size in sectors, in case of thin provisioning """ dom = sdCache.produce(sdUUID) dom.validateCreateVolumeParams(volFormat, srcVolUUID, preallocate=preallocate) imgPath = image.Image(repoPath).create(sdUUID, imgUUID) volPath = os.path.join(imgPath, volUUID) volParent = None volType = sc.type2name(sc.LEAF_VOL) # Get the specific class name and class module to be used in the # Recovery tasks. clsModule, clsName = cls._getModuleAndClass() try: if srcVolUUID != sc.BLANK_UUID: # When the srcImgUUID isn't specified we assume it's the same # as the imgUUID if srcImgUUID == sc.BLANK_UUID: srcImgUUID = imgUUID volParent = cls(repoPath, sdUUID, srcImgUUID, srcVolUUID) if not volParent.isLegal(): raise se.createIllegalVolumeSnapshotError( volParent.volUUID) if imgUUID != srcImgUUID: volParent.share(imgPath) volParent = cls(repoPath, sdUUID, imgUUID, srcVolUUID) # Override the size with the size of the parent size = volParent.getSize() except se.StorageException: cls.log.error("Unexpected error", exc_info=True) raise except Exception as e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumeCannotGetParent( "Couldn't get parent %s for volume %s: %s" % (srcVolUUID, volUUID, e)) try: cls.log.info("Creating volume %s", volUUID) # Rollback sentinel to mark the start of the task vars.task.pushRecovery( task.Recovery(task.ROLLBACK_SENTINEL, clsModule, clsName, "startCreateVolumeRollback", [sdUUID, imgUUID, volUUID])) # Create volume rollback vars.task.pushRecovery( task.Recovery("Halfbaked volume rollback", clsModule, clsName, "halfbakedVolumeRollback", [sdUUID, volUUID, volPath])) # Specific volume creation (block, file, etc...) try: metaId = cls._create(dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=initialSize) except (se.VolumeAlreadyExists, se.CannotCreateLogicalVolume, se.VolumeCreationError, se.InvalidParameterException) as e: cls.log.error("Failed to create volume %s: %s", volPath, e) vars.task.popRecovery() raise # When the volume format is raw what the guest sees is the apparent # size of the file/device therefore if the requested size doesn't # match the apparent size (eg: physical extent granularity in LVM) # we need to update the size value so that the metadata reflects # the correct state. if volFormat == sc.RAW_FORMAT: apparentSize = int( dom.getVSize(imgUUID, volUUID) / sc.BLOCK_SIZE) if apparentSize < size: cls.log.error( "The volume %s apparent size %s is smaller " "than the requested size %s", volUUID, apparentSize, size) raise se.VolumeCreationError() if apparentSize > size: cls.log.info( "The requested size for volume %s doesn't " "match the granularity on domain %s, " "updating the volume size from %s to %s", volUUID, sdUUID, size, apparentSize) size = apparentSize vars.task.pushRecovery( task.Recovery("Create volume metadata rollback", clsModule, clsName, "createVolumeMetadataRollback", map(str, metaId))) cls.newMetadata(metaId, sdUUID, imgUUID, srcVolUUID, size, sc.type2name(volFormat), sc.type2name(preallocate), volType, diskType, desc, sc.LEGAL_VOL) if dom.hasVolumeLeases(): cls.newVolumeLease(metaId, sdUUID, volUUID) except se.StorageException: cls.log.error("Unexpected error", exc_info=True) raise except Exception as e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumeCreationError("Volume creation %s failed: %s" % (volUUID, e)) # Remove the rollback for the halfbaked volume vars.task.replaceRecoveries( task.Recovery("Create volume rollback", clsModule, clsName, "createVolumeRollback", [repoPath, sdUUID, imgUUID, volUUID, imgPath])) return volUUID