def createVolumeMetadataRollback(cls, taskObj, volPath): cls.log.info("createVolumeMetadataRollback: volPath=%s" % (volPath)) metaPath = cls.manifestClass.metaVolumePath(volPath) sdUUID = getDomUuidFromVolumePath(volPath) if oop.getProcessPool(sdUUID).os.path.lexists(metaPath): cls.log.info("Unlinking metadata volume %r", metaPath) oop.getProcessPool(sdUUID).os.unlink(metaPath)
def create(cls, sdUUID, domainName, domClass, remotePath, storageType, version, block_size=sc.BLOCK_SIZE_512, alignment=sc.ALIGNMENT_1M): """ Create new storage domain Arguments: sdUUID (UUID): Storage Domain UUID domainName (str): Storage domain name domClass (int): Data/Iso remotePath (str): server:/export_path storageType (int): NFS_DOMAIN, GLUSTERFS_DOMAIN, &etc. version (int): DOMAIN_VERSIONS, block_size (int): Underlying storage block size. Supported value is BLOCK_SIZE_512 alignment (int): Sanlock alignment in bytes to use for this storage domain. Supported value is ALIGN_1M """ cls.log.info("sdUUID=%s domainName=%s remotePath=%s " "domClass=%s, block_size=%s, alignment=%s", sdUUID, domainName, remotePath, domClass, block_size, alignment) cls._validate_block_and_alignment(block_size, alignment, version) remotePath = fileUtils.normalize_path(remotePath) if not misc.isAscii(domainName) and not sd.supportsUnicode(version): raise se.UnicodeArgumentException() # Create local path mntPath = fileUtils.transformPath(remotePath) mntPoint = cls.getMountPoint(mntPath) cls._preCreateValidation(sdUUID, mntPoint, remotePath, storageType, version) domainDir = os.path.join(mntPoint, sdUUID) cls._prepareMetadata(domainDir, sdUUID, domainName, domClass, remotePath, storageType, version, alignment, block_size) # create domain images folder imagesDir = os.path.join(domainDir, sd.DOMAIN_IMAGES) cls.log.info("Creating domain images directory %r", imagesDir) oop.getProcessPool(sdUUID).fileUtils.createdir(imagesDir) # create special imageUUID for ISO/Floppy volumes if domClass is sd.ISO_DOMAIN: isoDir = os.path.join(imagesDir, sd.ISO_IMAGE_UUID) cls.log.info("Creating ISO domain images directory %r", isoDir) oop.getProcessPool(sdUUID).fileUtils.createdir(isoDir) fsd = cls(os.path.join(mntPoint, sdUUID)) fsd.initSPMlease() return fsd
def file_setrw(cls, volPath, rw): sdUUID = getDomUuidFromVolumePath(volPath) mode = 0o440 if rw: mode |= 0o220 if oop.getProcessPool(sdUUID).os.path.isdir(volPath): mode |= 0o110 oop.getProcessPool(sdUUID).os.chmod(volPath, mode)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ if initialSize: cls.log.error("initialSize is not supported for file-based " "volumes") raise se.InvalidParameterException("initial size", initialSize) sizeBytes = size * BLOCK_SIZE truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0 try: oop.getProcessPool(dom.sdUUID).truncateFile( volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == sc.PREALLOCATED_VOL: try: operation = fallocate.allocate(volPath, sizeBytes) with vars.task.abort_callback(operation.abort): with utils.stopwatch("Preallocating volume %s" % volPath): operation.run() except exception.ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info("Request to create %s volume %s with size = %s " "sectors", sc.type2name(volFormat), volPath, size) if volFormat == sc.COW_FORMAT: qemuimg.create(volPath, size=sizeBytes, format=sc.fmt2str(volFormat), qcow2Compat=dom.qcow2_compat()) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(volPath, volFormat) # Forcing the volume permissions in case one of the tools we use # (dd, qemu-img, etc.) will mistakenly change the file permissiosn. dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS) return (volPath,)
def newVolumeLease(cls, metaId, sdUUID, volUUID): cls.log.debug("Initializing volume lease volUUID=%s sdUUID=%s, " "metaId=%s", volUUID, sdUUID, metaId) volPath, = metaId leasePath = cls.leaseVolumePath(volPath) oop.getProcessPool(sdUUID).truncateFile(leasePath, LEASE_FILEOFFSET) cls.file_setrw(leasePath, rw=True) sanlock.init_resource(sdUUID, volUUID, [(leasePath, LEASE_FILEOFFSET)])
def renameVolumeRollback(cls, taskObj, oldPath, newPath): try: cls.log.info("oldPath=%s newPath=%s", oldPath, newPath) sdUUID = getDomUuidFromVolumePath(oldPath) oop.getProcessPool(sdUUID).os.rename(oldPath, newPath) except Exception: cls.log.error("Could not rollback " "volume rename (oldPath=%s newPath=%s)", oldPath, newPath, exc_info=True)
def _truncate_volume(cls, vol_path, size, vol_id, dom): try: oop.getProcessPool(dom.sdUUID).truncateFile( vol_path, size, mode=sc.FILE_VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(vol_id) raise
def _putMetadata(cls, metaId, meta): volPath, = metaId metaPath = cls.metaVolumePath(volPath) data = cls.formatMetadata(meta) with open(metaPath + ".new", "w") as f: f.write(data) sdUUID = getDomUuidFromVolumePath(volPath) oop.getProcessPool(sdUUID).os.rename(metaPath + ".new", metaPath)
def _putMetadata(cls, metaId, meta, **overrides): volPath, = metaId metaPath = cls.metaVolumePath(volPath) sd = sdCache.produce_manifest(meta.domain) data = meta.storage_format(sd.getVersion(), **overrides) with open(metaPath + ".new", "w") as f: f.write(data) oop.getProcessPool(meta.domain).os.rename(metaPath + ".new", metaPath)
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version, alignment, block_size): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) procPool = oop.getProcessPool(sdUUID) cls.log.info("Creating domain metadata directory %r", metadataDir) procPool.fileUtils.createdir(metadataDir, 0o775) special_volumes = cls.manifestClass.special_volumes(version) for name, size_mb in FILE_SPECIAL_VOLUME_SIZES_MIB.iteritems(): if name in special_volumes: try: procPool.truncateFile( os.path.join(metadataDir, name), size_mb * constants.MEGAB, METADATA_PERMISSIONS) except Exception as e: raise se.StorageDomainMetadataCreationError( "create meta file '%s' failed: %s" % (name, str(e))) if cls.supports_external_leases(version): xleases_path = os.path.join(metadataDir, sd.XLEASES) cls.format_external_leases(sdUUID, xleases_path) metaFile = os.path.join(metadataDir, sd.METADATA) md = FileSDMetadata(metaFile) # initialize domain metadata content # FIXME : This is 99% like the metadata in block SD # Do we really need to keep the EXPORT_PATH? # no one uses it metadata = { sd.DMDK_VERSION: version, sd.DMDK_SDUUID: sdUUID, sd.DMDK_TYPE: storageType, sd.DMDK_CLASS: domClass, sd.DMDK_DESCRIPTION: domainName, sd.DMDK_ROLE: sd.REGULAR_DOMAIN, sd.DMDK_POOLS: [], sd.DMDK_LOCK_POLICY: '', sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_LEASE_TIME_SEC: sd.DEFAULT_LEASE_PARAMS[ sd.DMDK_LEASE_TIME_SEC], sd.DMDK_IO_OP_TIMEOUT_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_IO_OP_TIMEOUT_SEC], sd.DMDK_LEASE_RETRIES: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_RETRIES], REMOTE_PATH: remotePath } if version > 4: metadata[sd.DMDK_ALIGNMENT] = alignment metadata[sd.DMDK_BLOCK_SIZE] = block_size md.update(metadata)
def halfbakedVolumeRollback(cls, taskObj, *args): if len(args) == 1: # Backward compatibility volPath, = args sdUUID = getDomUuidFromVolumePath(volPath) elif len(args) == 3: (sdUUID, volUUID, volPath) = args else: raise TypeError("halfbakedVolumeRollback takes 1 or 3 " "arguments (%d given)" % len(args)) metaVolPath = cls.manifestClass.metaVolumePath(volPath) cls.log.info("Halfbaked volume rollback for volPath=%s", volPath) if oop.getProcessPool(sdUUID).fileUtils.pathExists(volPath) and not \ oop.getProcessPool(sdUUID).fileUtils.pathExists(metaVolPath): oop.getProcessPool(sdUUID).os.unlink(volPath)
def collectMetaFiles(mountPoint): try: # removes the path to the data center's mount directory from # the mount point. if mountPoint.startswith(sc.REPO_MOUNT_DIR): client_name = mountPoint[len(sc.REPO_MOUNT_DIR):] # Since glob treats values between brackets as character ranges, # and since IPV6 addresses contain brackets, we should escape the # mountPoint that we pass to glob. # <data-center>/mnt/mountpoint/<uuid>/dom_mdm mdPattern = os.path.join( glob_escape(mountPoint), UUID_GLOB_PATTERN, sd.DOMAIN_META_DATA) metaFiles = oop.getProcessPool(client_name).glob.glob(mdPattern) for metaFile in metaFiles: if (os.path.basename(os.path.dirname(metaFile)) != sd.MASTER_FS_DIR): sdUUID = os.path.basename(os.path.dirname(metaFile)) return (sdUUID, os.path.dirname(metaFile)) except Exception: log.warn("Could not collect metadata file for domain path %s", mountPoint, exc_info=True)
def validateFileSystemFeatures(sdUUID, mountDir): try: # Don't unlink this file, we don't have the cluster lock yet as it # requires direct IO which is what we are trying to test for. This # means that unlinking the file might cause a race. Since we don't # care what the content of the file is, just that we managed to # open it O_DIRECT. testFilePath = os.path.join(mountDir, "__DIRECT_IO_TEST__") oop.getProcessPool(sdUUID).directTouch(testFilePath) except OSError as e: if e.errno == errno.EINVAL: log = logging.getLogger("storage.fileSD") log.error("Underlying file system doesn't support" "direct IO") raise se.StorageDomainTargetUnsupported() raise
def format(cls, sdUUID): """ Format detached storage domain. This removes all data from the storage domain. """ cls.log.info("Formatting domain %s", sdUUID) try: domaindir = cls.findDomainPath(sdUUID) except (se.StorageDomainDoesNotExist): pass else: try: oop.getProcessPool(sdUUID).fileUtils.cleanupdir( domaindir, ignoreErrors=False) except RuntimeError as e: raise se.MiscDirCleanupFailure(str(e)) return True
def testSamePoolName(self): poolA = "A" pids = [] for pool in (poolA, poolA): proc = oop.getProcessPool(pool)._ioproc name = proc._commthread.getName() pids.append(int(re.search(r'\d+', name).group())) self.assertEqual(pids[0], pids[1])
def create(cls, sdUUID, domainName, domClass, remotePath, storageType, version): """ Create new storage domain. 'sdUUID' - Storage Domain UUID 'domainName' - storage domain name ("iso" or "data domain name") 'domClass' - Data/Iso 'remotePath' - server:/export_path 'storageType' - NFS_DOMAIN, LOCALFS_DOMAIN, &etc. 'version' - DOMAIN_VERSIONS """ cls.log.info("sdUUID=%s domainName=%s remotePath=%s " "domClass=%s", sdUUID, domainName, remotePath, domClass) remotePath = fileUtils.normalize_path(remotePath) if not misc.isAscii(domainName) and not sd.supportsUnicode(version): raise se.UnicodeArgumentException() # Create local path mntPath = fileUtils.transformPath(remotePath) mntPoint = cls.getMountPoint(mntPath) cls._preCreateValidation(sdUUID, mntPoint, remotePath, storageType, version) domainDir = os.path.join(mntPoint, sdUUID) cls._prepareMetadata(domainDir, sdUUID, domainName, domClass, remotePath, storageType, version) # create domain images folder imagesDir = os.path.join(domainDir, sd.DOMAIN_IMAGES) oop.getProcessPool(sdUUID).fileUtils.createdir(imagesDir) # create special imageUUID for ISO/Floppy volumes if domClass is sd.ISO_DOMAIN: isoDir = os.path.join(imagesDir, sd.ISO_IMAGE_UUID) oop.getProcessPool(sdUUID).fileUtils.createdir(isoDir) fsd = cls(os.path.join(mntPoint, sdUUID)) fsd.initSPMlease() return fsd
def setMetadataPermissions(self): procPool = oop.getProcessPool(self.sdUUID) for metaFile in (sd.LEASES, sd.IDS, sd.INBOX, sd.OUTBOX): try: fpath = os.path.join(self.getMDPath(), metaFile) procPool.os.chmod(fpath, METADATA_PERMISSIONS) except Exception as e: raise se.StorageDomainMetadataCreationError( "Lease permission change file '%s' failed: %s" % (metaFile, e))
def testDifferentPoolName(self): poolA = "A" poolB = "B" pools = (poolA, poolB) pids = [] for pool in pools: proc = oop.getProcessPool(pool)._ioproc name = proc._commthread.name pids.append(int(re.search(r'\d+', name).group())) self.assertNotEquals(pids[0], pids[1])
def external_leases_backend(cls, lockspace, path): """ Overrides StorageDomainManifest method to use an interruptible direct file implementation that will not make the processs uninterruptible if the storage becomes non-responsive. See https://bugzilla.redhat.com/1518676. See StorageDomainManifest.external_leases_backend for more info. """ dom_oop = oop.getProcessPool(lockspace) backend = xlease.InterruptibleDirectFile(path, dom_oop) with utils.closing(backend): yield backend
def direct_file(request): """ Returns a direct file factory function accpting a path. Test for xlease.*DirectFile can use this fixture for testing both implemntations. """ if request.param == xlease.InterruptibleDirectFile: try: test_oop = oop.getProcessPool("test") yield functools.partial(request.param, oop=test_oop) finally: oop.stop() else: yield request.param
def getImageVolumes(cls, sdUUID, imgUUID): """ Fetch the list of the Volumes UUIDs, not including the shared base (template) """ sd = sdCache.produce_manifest(sdUUID) img_dir = sd.getImageDir(imgUUID) pattern = os.path.join(glob_escape(img_dir), "*.meta") files = oop.getProcessPool(sdUUID).glob.glob(pattern) volList = [] for i in files: volid = os.path.splitext(os.path.basename(i))[0] if (sd.produceVolume(imgUUID, volid).getImage() == imgUUID): volList.append(volid) return volList
def create_external_leases(self): """ Create the external leases special volume. Called during upgrade from version 3 to version 4. """ proc = oop.getProcessPool(self.sdUUID) path = self.external_leases_path() size = FILE_SPECIAL_VOLUME_SIZES_MIB[sd.XLEASES] * constants.MEGAB self.log.info("Creating external leases volume %s", path) try: proc.truncateFile(path, size, METADATA_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: self.log.info("Reusing external leases volume %s", path)
def getImageVolumes(cls, repoPath, sdUUID, imgUUID): """ Fetch the list of the Volumes UUIDs, not including the shared base (template) """ # Get Volumes of an image pattern = os.path.join(repoPath, sdUUID, sd.DOMAIN_IMAGES, imgUUID, "*.meta") files = oop.getProcessPool(sdUUID).glob.glob(pattern) volList = [] for i in files: volid = os.path.splitext(os.path.basename(i))[0] if (sdCache.produce(sdUUID). produceVolume(imgUUID, volid). getImage() == imgUUID): volList.append(volid) return volList
def collectMetaFiles(possibleDomain): try: metaFiles = oop.getProcessPool(possibleDomain).glob.glob( os.path.join(possibleDomain, constants.UUID_GLOB_PATTERN, sd.DOMAIN_META_DATA)) for metaFile in metaFiles: if (os.path.basename(os.path.dirname(metaFile)) != sd.MASTER_FS_DIR): sdUUID = os.path.basename(os.path.dirname(metaFile)) return (sdUUID, os.path.dirname(metaFile)) except Exception: log.warn("Could not collect metadata file for domain path %s", possibleDomain, exc_info=True)
def getChildren(self): """ Return children volume UUIDs. This API is not suitable for use with a template's base volume. """ domPath = self.imagePath.split('images')[0] metaPattern = os.path.join(domPath, 'images', self.imgUUID, '*.meta') metaPaths = oop.getProcessPool(self.sdUUID).glob.glob(metaPattern) pattern = "%s.*%s" % (sc.PUUID, self.volUUID) matches = grepCmd(pattern, metaPaths) if matches: children = [] for line in matches: volMeta = os.path.basename(line.rsplit(':', 1)[0]) children.append(os.path.splitext(volMeta)[0]) # volUUID else: children = tuple() return tuple(children)
def collectMetaFiles(possibleDomain): try: # Since glob treats values between brackets as character ranges, # and since IPV6 addresses contain brackets, we should escape the # possibleDomain that we pass to glob. metaFiles = oop.getProcessPool(possibleDomain).glob.glob( os.path.join(glob_escape(possibleDomain), constants.UUID_GLOB_PATTERN, sd.DOMAIN_META_DATA)) for metaFile in metaFiles: if (os.path.basename(os.path.dirname(metaFile)) != sd.MASTER_FS_DIR): sdUUID = os.path.basename(os.path.dirname(metaFile)) return (sdUUID, os.path.dirname(metaFile)) except Exception: log.warn("Could not collect metadata file for domain path %s", possibleDomain, exc_info=True)
def _preCreateValidation(cls, sdUUID, domPath, typeSpecificArg, storageType, version): # Some trivial resource validation # TODO Checking storageType==nfs in the nfs class is not clean if storageType == sd.NFS_DOMAIN and ":" not in typeSpecificArg: raise se.StorageDomainIllegalRemotePath(typeSpecificArg) sd.validateDomainVersion(version) # Make sure the underlying file system is mounted if not mount.isMounted(domPath): raise se.StorageDomainFSNotMounted(domPath) fileSD.validateDirAccess(domPath) fileSD.validateFileSystemFeatures(sdUUID, domPath) # Make sure there are no remnants of other domain mdpat = os.path.join(domPath, "*", sd.DOMAIN_META_DATA) if len(oop.getProcessPool(sdUUID).glob.glob(mdpat)) > 0: raise se.StorageDomainNotEmpty(typeSpecificArg)
def collectMetaFiles(possibleDomain): try: # Since glob treats values between brackets as character ranges, # and since IPV6 addresses contain brackets, we should escape the # possibleDomain that we pass to glob. metaFiles = oop.getProcessPool(possibleDomain).glob.glob( os.path.join(glob_escape(possibleDomain), UUID_GLOB_PATTERN, sd.DOMAIN_META_DATA)) for metaFile in metaFiles: if (os.path.basename(os.path.dirname(metaFile)) != sd.MASTER_FS_DIR): sdUUID = os.path.basename(os.path.dirname(metaFile)) return (sdUUID, os.path.dirname(metaFile)) except Exception: log.warn("Could not collect metadata file for domain path %s", possibleDomain, exc_info=True)
def _detect_block_size(cls, sd_id, mountpoint): """ Detect filesystem block size and validate direct I/O usage. There is no way to get the underlying storage logical block size, but since direct I/O must be aligned to the logical block size, we can detect the capability by trying to write a file using direct I/O. This creates a file __DIRECT_IO_TEST__ in the mountpoint. Removing this file is racy so we leave it. Raises: se.StorageDomainTargetUnsupported if direct I/O fails for all supported block sizes. OSError if writing failed because of another error. ioprocess.Timeout if writing to storage timed out. Returns: the detected block size (1, 512, 4096) """ log = logging.getLogger("storage.fileSD") path = os.path.join(mountpoint, "__DIRECT_IO_TEST__") iop = oop.getProcessPool(sd_id) for block_size in (sc.BLOCK_SIZE_NONE, sc.BLOCK_SIZE_512, sc.BLOCK_SIZE_4K): log.debug("Trying block size %s", block_size) data = b"\0" * block_size try: iop.writeFile(path, data, direct=True) except OSError as e: if e.errno != errno.EINVAL: raise else: log.debug("Detected domain %s block size %s", sd_id, block_size) return block_size raise se.StorageDomainTargetUnsupported( "Failed to write to {} using direct I/O".format(path))
def testAmountOfInstancesPerPoolName(self): with MonkeyPatchScope([(oop, 'IOPROC_IDLE_TIME', 0.5)]): poolA = "A" poolB = "B" wrapper = ref(oop.getProcessPool(poolA)) ioproc = ref(oop.getProcessPool(poolA)._ioproc) oop.getProcessPool(poolA) time.sleep(oop.IOPROC_IDLE_TIME + 0.5) oop.getProcessPool(poolB) self.assertEqual(wrapper(), None) gc.collect() try: self.assertEqual(ioproc(), None) except AssertionError: logging.info("GARBAGE: %s", gc.garbage) refs = gc.get_referrers(ioproc()) logging.info(refs) logging.info(gc.get_referrers(*refs)) raise
def testAmountOfInstancesPerPoolName(self, monkeypatch): monkeypatch.setattr(oop, 'IOPROC_IDLE_TIME', 0.5) poolA = "A" poolB = "B" wrapper = ref(oop.getProcessPool(poolA)) ioproc = ref(oop.getProcessPool(poolA)._ioproc) oop.getProcessPool(poolA) time.sleep(oop.IOPROC_IDLE_TIME + 0.5) oop.getProcessPool(poolB) assert wrapper() is None gc.collect() try: assert ioproc() is None except AssertionError: logging.info("GARBAGE: %s", gc.garbage) refs = gc.get_referrers(ioproc()) logging.info(refs) logging.info(gc.get_referrers(*refs)) raise
def shareVolumeRollback(cls, taskObj, volPath): cls.log.info("Volume rollback for volPath=%s", volPath) procPool = oop.getProcessPool(getDomUuidFromVolumePath(volPath)) procPool.utils.rmFile(volPath) procPool.utils.rmFile(cls.manifestClass.metaVolumePath(volPath)) procPool.utils.rmFile(cls.manifestClass.leaseVolumePath(volPath))
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ if initialSize: cls.log.error("initialSize is not supported for file-based " "volumes") raise se.InvalidParameterException("initial size", initialSize) sizeBytes = size * BLOCK_SIZE truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0 try: oop.getProcessPool(dom.sdUUID).truncateFile( volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == sc.PREALLOCATED_VOL: try: operation = fallocate.allocate(volPath, sizeBytes) with vars.task.abort_callback(operation.abort): with utils.stopwatch("Preallocating volume %s" % volPath): operation.run() except exception.ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info( "Request to create %s volume %s with size = %s " "blocks", sc.type2name(volFormat), volPath, size) if volFormat == sc.COW_FORMAT: operation = qemuimg.create(volPath, size=sizeBytes, format=sc.fmt2str(volFormat), qcow2Compat=dom.qcow2_compat()) operation.run() else: # Create hardlink to template and its meta file cls.log.info( "Request to create snapshot %s/%s of volume %s/%s " "with size %s (blocks)", imgUUID, volUUID, srcImgUUID, srcVolUUID, size) volParent.clone(volPath, volFormat, size) # Forcing the volume permissions in case one of the tools we use # (dd, qemu-img, etc.) will mistakenly change the file permissiosn. cls.log.info("Changing volume %r permission to %04o", volPath, sc.FILE_VOLUME_PERMISSIONS) dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS) return (volPath, )
def oop_ns(): try: yield oop.getProcessPool("test") finally: oop.stop()
def test_fileutils_pathexists(oop_cleanup): iop = oop.getProcessPool("test") path = "/dev/null" assert iop.fileUtils.pathExists(path)
def create(cls, sdUUID, domainName, domClass, remotePath, storageType, version, block_size=sc.BLOCK_SIZE_512, alignment=sc.ALIGN_1M): """ Create new storage domain Arguments: sdUUID (UUID): Storage Domain UUID domainName (str): Storage domain name domClass (int): Data/Iso remotePath (str): server:/export_path storageType (int): NFS_DOMAIN, GLUSTERFS_DOMAIN, &etc. version (int): DOMAIN_VERSIONS, block_size (int): Underlying storage block size. Supported value is BLOCK_SIZE_512 alignment (int): Sanlock alignment in bytes to use for this storage domain. Supported value is ALIGN_1M """ cls.log.info( "sdUUID=%s domainName=%s remotePath=%s " "domClass=%s, block_size=%s, alignment=%s", sdUUID, domainName, remotePath, domClass, block_size, alignment) cls._validate_block_and_alignment(block_size, alignment) remotePath = fileUtils.normalize_path(remotePath) if not misc.isAscii(domainName) and not sd.supportsUnicode(version): raise se.UnicodeArgumentException() # Create local path mntPath = fileUtils.transformPath(remotePath) mntPoint = cls.getMountPoint(mntPath) cls._preCreateValidation(sdUUID, mntPoint, remotePath, storageType, version) domainDir = os.path.join(mntPoint, sdUUID) cls._prepareMetadata(domainDir, sdUUID, domainName, domClass, remotePath, storageType, version) # create domain images folder imagesDir = os.path.join(domainDir, sd.DOMAIN_IMAGES) cls.log.info("Creating domain images directory %r", imagesDir) oop.getProcessPool(sdUUID).fileUtils.createdir(imagesDir) # create special imageUUID for ISO/Floppy volumes if domClass is sd.ISO_DOMAIN: isoDir = os.path.join(imagesDir, sd.ISO_IMAGE_UUID) cls.log.info("Creating ISO domain images directory %r", isoDir) oop.getProcessPool(sdUUID).fileUtils.createdir(isoDir) fsd = cls(os.path.join(mntPoint, sdUUID)) fsd.initSPMlease() return fsd
def test_os_path_islink(oop_cleanup, tmpdir): iop = oop.getProcessPool("test") link = str(tmpdir.join("link")) os.symlink("/no/such/file", link) assert iop.os.path.islink(link)
def test_truncate_file_non_default_mode(oop_cleanup, tmpdir, mode): iop = oop.getProcessPool("test") path = str(tmpdir.join("file")) iop.truncateFile(path, 10, mode) verify_file(path, mode, size=10, content=b"\0" * 10)
def __init__(self, metafile): # FileSDMetadata is kept in the file self._metafile = metafile self._sdUUID = getDomUuidFromMetafilePath(metafile) self._oop = oop.getProcessPool(self._sdUUID)
def createVolumeMetadataRollback(cls, taskObj, volPath): cls.log.info("createVolumeMetadataRollback: volPath=%s" % (volPath)) metaPath = cls.manifestClass.metaVolumePath(volPath) sdUUID = getDomUuidFromVolumePath(volPath) if oop.getProcessPool(sdUUID).os.path.lexists(metaPath): oop.getProcessPool(sdUUID).os.unlink(metaPath)
def test_os_path_exists(oop_cleanup): path = "/dev/null" iop = oop.getProcessPool("test") assert iop.os.path.exists(path)
def create(cls, sdUUID, domainName, domClass, remotePath, storageType, version, block_size=sc.BLOCK_SIZE_512, max_hosts=sc.HOSTS_4K_1M): """ Create new storage domain Arguments: sdUUID (UUID): Storage Domain UUID domainName (str): Storage domain name domClass (int): Data/Iso remotePath (str): server:/export_path storageType (int): NFS_DOMAIN, GLUSTERFS_DOMAIN, &etc. version (int): DOMAIN_VERSIONS, block_size (int): Underlying storage block size. Supported value is BLOCK_SIZE_512 max_hosts (int): Maximum number of hosts accessing this domain, default to sc.HOSTS_4K_1M. """ cls._validate_block_size(block_size, version) remotePath = fileUtils.normalize_path(remotePath) if not misc.isAscii(domainName) and not sd.supportsUnicode(version): raise se.UnicodeArgumentException() # Create local path mntPath = fileUtils.transformPath(remotePath) mntPoint = cls.getMountPoint(mntPath) cls._preCreateValidation(sdUUID, mntPoint, remotePath, storageType, version) storage_block_size = cls._detect_block_size(sdUUID, mntPoint) block_size = cls._validate_storage_block_size(block_size, storage_block_size) alignment = clusterlock.alignment(block_size, max_hosts) domainDir = os.path.join(mntPoint, sdUUID) cls._prepareMetadata(domainDir, sdUUID, domainName, domClass, remotePath, storageType, version, alignment, block_size) # create domain images folder imagesDir = os.path.join(domainDir, sd.DOMAIN_IMAGES) cls.log.info("Creating domain images directory %r", imagesDir) oop.getProcessPool(sdUUID).fileUtils.createdir(imagesDir) # create special imageUUID for ISO/Floppy volumes if domClass is sd.ISO_DOMAIN: isoDir = os.path.join(imagesDir, sd.ISO_IMAGE_UUID) cls.log.info("Creating ISO domain images directory %r", isoDir) oop.getProcessPool(sdUUID).fileUtils.createdir(isoDir) fsd = cls(os.path.join(mntPoint, sdUUID)) fsd.initSPMlease() return fsd
def pad_memory_volume(memory_vol_path, sd_uuid): sd_type = sd.name2type( self.vm.cif.irs.getStorageDomainInfo(sd_uuid)['info']['type']) if sd_type in sd.FILE_DOMAIN_TYPES: iop = oop.getProcessPool(sd_uuid) iop.fileUtils.padToBlockSize(memory_vol_path)
def oop(self): return oop.getProcessPool(self.sdUUID)
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version, alignment, block_size): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) procPool = oop.getProcessPool(sdUUID) cls.log.info("Creating domain metadata directory %r", metadataDir) procPool.fileUtils.createdir(metadataDir, 0o775) special_volumes = cls.manifestClass.special_volumes(version) for name, size_mb in six.iteritems(FILE_SPECIAL_VOLUME_SIZES_MIB): if name in special_volumes: try: procPool.truncateFile(os.path.join(metadataDir, name), size_mb * MiB, METADATA_PERMISSIONS) except Exception as e: raise se.StorageDomainMetadataCreationError( "create meta file '%s' failed: %s" % (name, str(e))) if cls.supports_external_leases(version): xleases_path = os.path.join(metadataDir, sd.XLEASES) cls.format_external_leases(sdUUID, xleases_path, alignment=alignment, block_size=block_size) metaFile = os.path.join(metadataDir, sd.METADATA) md = FileSDMetadata(metaFile) # initialize domain metadata content # FIXME : This is 99% like the metadata in block SD # Do we really need to keep the EXPORT_PATH? # no one uses it metadata = { sd.DMDK_VERSION: version, sd.DMDK_SDUUID: sdUUID, sd.DMDK_TYPE: storageType, sd.DMDK_CLASS: domClass, sd.DMDK_DESCRIPTION: domainName, sd.DMDK_ROLE: sd.REGULAR_DOMAIN, sd.DMDK_POOLS: [], sd.DMDK_LOCK_POLICY: '', sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_LEASE_TIME_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_TIME_SEC], sd.DMDK_IO_OP_TIMEOUT_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_IO_OP_TIMEOUT_SEC], sd.DMDK_LEASE_RETRIES: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_RETRIES], REMOTE_PATH: remotePath } if version > 4: metadata[sd.DMDK_ALIGNMENT] = alignment metadata[sd.DMDK_BLOCK_SIZE] = block_size md.update(metadata)
def test_sub_module_call(oop_cleanup): path = "/dev/null" iop = oop.getProcessPool("test") assert iop.os.path.exists(path)
def test_fileutils_call(oop_cleanup): """fileUtils is a custom module and calling it might break even though built in module calls aren't broken""" iop = oop.getProcessPool("test") path = "/dev/null" assert iop.fileUtils.pathExists(path)
def test_os_path_islink_not_link(oop_cleanup, tmpdir): iop = oop.getProcessPool("test") assert not iop.os.path.islink(str(tmpdir))
def getProcPool(): return oop.getProcessPool(sc.GLOBAL_OOP)