def newVolumeLease(cls, metaId, sdUUID, volUUID): cls.log.debug("Initializing volume lease volUUID=%s sdUUID=%s, " "metaId=%s", volUUID, sdUUID, metaId) volPath, = metaId leasePath = cls.__leaseVolumePath(volPath) oop.getProcessPool(sdUUID).truncateFile(leasePath, LEASE_FILEOFFSET) cls.file_setrw(leasePath, rw=True) sanlock.init_resource(sdUUID, volUUID, [(leasePath, LEASE_FILEOFFSET)])
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) oop.getProcessPool(sdUUID).fileUtils.createdir(metadataDir, 0775) createmetafile(os.path.join(metadataDir, sd.LEASES), sd.LEASES_SIZE) createmetafile(os.path.join(metadataDir, sd.IDS), sd.IDS_SIZE) createmetafile(os.path.join(metadataDir, sd.INBOX), sd.INBOX_SIZE) createmetafile(os.path.join(metadataDir, sd.OUTBOX), sd.OUTBOX_SIZE) metaFile = os.path.join(metadataDir, sd.METADATA) md = FileSDMetadata(metaFile) # initialize domain metadata content # FIXME : This is 99% like the metadata in block SD # Do we really need to keep the EXPORT_PATH? # no one uses it md.update({ sd.DMDK_VERSION : version, sd.DMDK_SDUUID : sdUUID, sd.DMDK_TYPE : storageType, sd.DMDK_CLASS : domClass, sd.DMDK_DESCRIPTION : domainName, sd.DMDK_ROLE : sd.REGULAR_DOMAIN, sd.DMDK_POOLS : [], sd.DMDK_LOCK_POLICY : '', sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC : sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_LEASE_TIME_SEC : sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_IO_OP_TIMEOUT_SEC : sd.DEFAULT_LEASE_PARAMS[sd.DMDK_IO_OP_TIMEOUT_SEC], sd.DMDK_LEASE_RETRIES : sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_RETRIES], REMOTE_PATH : remotePath })
def renameVolumeRollback(cls, taskObj, oldPath, newPath): try: cls.log.info("oldPath=%s newPath=%s", oldPath, newPath) sdUUID = getDomUuidFromVolumePath(oldPath) oop.getProcessPool(sdUUID).os.rename(oldPath, newPath) except Exception: cls.log.error("Could not rollback volume rename (oldPath=%s newPath=%s)", oldPath, newPath, exc_info=True)
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ if initialSize: cls.log.error("initialSize is not supported for file-based " "volumes") raise se.InvalidParameterException("initial size", initialSize) sizeBytes = size * BLOCK_SIZE truncSize = sizeBytes if volFormat == volume.RAW_FORMAT else 0 try: oop.getProcessPool(dom.sdUUID).truncateFile( volPath, truncSize, mode=VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == volume.PREALLOCATED_VOL: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting, sizeBytes) except ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info( "Request to create %s volume %s with size = %s " "sectors", volume.type2name(volFormat), volPath, size) if volFormat == volume.COW_FORMAT: qemuimg.create(volPath, sizeBytes, volume.fmt2str(volFormat)) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(volPath, volFormat) # Forcing the volume permissions in case one of the tools we use # (dd, qemu-img, etc.) will mistakenly change the file permissiosn. dom.oop.os.chmod(volPath, VOLUME_PERMISSIONS) return (volPath, )
def file_setrw(volPath, rw): sdUUID = getDomUuidFromVolumePath(volPath) mode = 0440 if rw: mode |= 0220 if oop.getProcessPool(sdUUID).os.path.isdir(volPath): mode |= 0110 oop.getProcessPool(sdUUID).os.chmod(volPath, mode)
def file_setrw(cls, volPath, rw): sdUUID = getDomUuidFromVolumePath(volPath) mode = 0o440 if rw: mode |= 0o220 if oop.getProcessPool(sdUUID).os.path.isdir(volPath): mode |= 0o110 oop.getProcessPool(sdUUID).os.chmod(volPath, mode)
def _create( cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None, ): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ if initialSize: cls.log.error("initialSize is not supported for file-based " "volumes") raise se.InvalidParameterException("initial size", initialSize) sizeBytes = size * BLOCK_SIZE truncSize = sizeBytes if volFormat == volume.RAW_FORMAT else 0 try: oop.getProcessPool(dom.sdUUID).truncateFile(volPath, truncSize, mode=VOLUME_PERMISSIONS, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == volume.PREALLOCATED_VOL: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting, sizeBytes) except ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info( "Request to create %s volume %s with size = %s " "sectors", volume.type2name(volFormat), volPath, size ) if volFormat == volume.COW_FORMAT: qemuimg.create(volPath, sizeBytes, volume.fmt2str(volFormat)) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(volPath, volFormat) # Forcing the volume permissions in case one of the tools we use # (dd, qemu-img, etc.) will mistakenly change the file permissiosn. dom.oop.os.chmod(volPath, VOLUME_PERMISSIONS) return (volPath,)
def newVolumeLease(cls, metaId, sdUUID, volUUID): cls.log.debug( "Initializing volume lease volUUID=%s sdUUID=%s, " "metaId=%s", volUUID, sdUUID, metaId) volPath, = metaId leasePath = cls.leaseVolumePath(volPath) oop.getProcessPool(sdUUID).truncateFile(leasePath, LEASE_FILEOFFSET) cls.file_setrw(leasePath, rw=True) sanlock.init_resource(sdUUID, volUUID, [(leasePath, LEASE_FILEOFFSET)])
def renameVolumeRollback(cls, taskObj, oldPath, newPath): try: cls.log.info("oldPath=%s newPath=%s", oldPath, newPath) sdUUID = getDomUuidFromVolumePath(oldPath) oop.getProcessPool(sdUUID).os.rename(oldPath, newPath) except Exception: cls.log.error("Could not rollback " "volume rename (oldPath=%s newPath=%s)", oldPath, newPath, exc_info=True)
def __putMetadata(cls, metaId, meta): volPath, = metaId metaPath = cls.__metaVolumePath(volPath) with open(metaPath + ".new", "w") as f: for key, value in meta.iteritems(): f.write("%s=%s\n" % (key.strip(), str(value).strip())) f.write("EOF\n") sdUUID = getDomUuidFromVolumePath(volPath) oop.getProcessPool(sdUUID).os.rename(metaPath + ".new", metaPath)
def _putMetadata(cls, metaId, meta): volPath, = metaId metaPath = cls.metaVolumePath(volPath) data = cls.formatMetadata(meta) with open(metaPath + ".new", "w") as f: f.write(data) sdUUID = getDomUuidFromVolumePath(volPath) oop.getProcessPool(sdUUID).os.rename(metaPath + ".new", metaPath)
def __putMetadata(cls, metaId, meta): volPath, = metaId metaPath = cls.__metaVolumePath(volPath) data = cls.formatMetadata(meta) with open(metaPath + ".new", "w") as f: f.write(data) sdUUID = getDomUuidFromVolumePath(volPath) oop.getProcessPool(sdUUID).os.rename(metaPath + ".new", metaPath)
def halfbakedVolumeRollback(cls, taskObj, *args): if len(args) == 1: # Backward compatibility volPath, = args sdUUID = getDomUuidFromVolumePath(volPath) elif len(args) == 3: (sdUUID, volUUID, volPath) = args else: raise TypeError("halfbakedVolumeRollback takes 1 or 3 " "arguments (%d given)" % len(args)) cls.log.info("Halfbaked volume rollback for volPath=%s", volPath) if oop.getProcessPool(sdUUID).fileUtils.pathExists(volPath): oop.getProcessPool(sdUUID).os.unlink(volPath)
def format(cls, sdUUID): """ Format detached storage domain. This removes all data from the storage domain. """ cls.log.info("Formating domain %s", sdUUID) try: domaindir = cls.findDomainPath(sdUUID) except (se.StorageDomainDoesNotExist): pass else: oop.getProcessPool(sdUUID).fileUtils.cleanupdir(domaindir, ignoreErrors = False) return True
def __putMetadata(cls, metaarr, vol_path): meta = cls.__metaVolumePath(vol_path) f = None try: f = open(meta + ".new", "w") for key, value in metaarr.iteritems(): f.write("%s=%s\n" % (key.strip(), str(value).strip())) f.write("EOF\n") finally: if f: f.close() sdUUID = getDomUuidFromVolumePath(vol_path) oop.getProcessPool(sdUUID).os.rename(meta + ".new", meta)
def format(cls, sdUUID): """ Format detached storage domain. This removes all data from the storage domain. """ cls.log.info("Formating domain %s", sdUUID) try: domaindir = cls.findDomainPath(sdUUID) except (se.StorageDomainDoesNotExist): pass else: oop.getProcessPool(sdUUID).fileUtils.cleanupdir(domaindir, ignoreErrors=False) return True
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) oop.getProcessPool(sdUUID).fileUtils.createdir(metadataDir, 0775) createmetafile(os.path.join(metadataDir, sd.LEASES), sd.LEASES_SIZE) createmetafile(os.path.join(metadataDir, sd.IDS), sd.IDS_SIZE) createmetafile(os.path.join(metadataDir, sd.INBOX), sd.INBOX_SIZE) createmetafile(os.path.join(metadataDir, sd.OUTBOX), sd.OUTBOX_SIZE) metaFile = os.path.join(metadataDir, sd.METADATA) md = FileSDMetadata(metaFile) # initialize domain metadata content # FIXME : This is 99% like the metadata in block SD # Do we really need to keep the EXPORT_PATH? # no one uses it md.update({ sd.DMDK_VERSION: version, sd.DMDK_SDUUID: sdUUID, sd.DMDK_TYPE: storageType, sd.DMDK_CLASS: domClass, sd.DMDK_DESCRIPTION: domainName, sd.DMDK_ROLE: sd.REGULAR_DOMAIN, sd.DMDK_POOLS: [], sd.DMDK_LOCK_POLICY: '', sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_LEASE_TIME_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_IO_OP_TIMEOUT_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_IO_OP_TIMEOUT_SEC], sd.DMDK_LEASE_RETRIES: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_RETRIES], REMOTE_PATH: remotePath })
def validateFileSystemFeatures(sdUUID, mountDir): try: # Don't unlink this file, we don't have the cluster lock yet as it # requires direct IO which is what we are trying to test for. This # means that unlinking the file might cause a race. Since we don't # care what the content of the file is, just that we managed to # open it O_DIRECT. testFilePath = os.path.join(mountDir, "__DIRECT_IO_TEST__") oop.getProcessPool(sdUUID).directTouch(testFilePath) except OSError as e: if e.errno == errno.EINVAL: log = logging.getLogger("Storage.fileSD") log.error("Underlying file system doesn't support" "direct IO") raise se.StorageDomainTargetUnsupported() raise
def getAllChildrenList(cls, repoPath, sdUUID, imgUUID, pvolUUID): """ Fetch the list of children volumes (across the all images in domain) """ volList = [] # FIXME!!! We cannot check hardlinks in 'backup' domain, because of possibility of overwriting # 'fake' volumes that have hardlinks with 'legal' volumes with same uuid and without hardlinks # First, check number of hardlinks ## volPath = os.path.join(cls.storage_repository, spUUID, sdUUID, sd.DOMAIN_IMAGES, imgUUID, pvolUUID) ## if os.path.exists(volPath): ## if os.stat(volPath).st_nlink == 1: ## return volList ## else: ## cls.log.info("Volume %s does not exist", volPath) ## return volList # scan whole domain pattern = os.path.join(repoPath, sdUUID, sd.DOMAIN_IMAGES, "*", "*.meta") files = oop.getProcessPool(sdUUID).glob.glob(pattern) for i in files: volid = os.path.splitext(os.path.basename(i))[0] imgUUID = os.path.basename(os.path.dirname(i)) if sdCache.produce(sdUUID).produceVolume( imgUUID, volid).getParent() == pvolUUID: volList.append({'imgUUID': imgUUID, 'volUUID': volid}) return volList
def getAllChildrenList(cls, repoPath, sdUUID, imgUUID, pvolUUID): """ Fetch the list of children volumes (across the all images in domain) """ volList = [] # FIXME!!! We cannot check hardlinks in 'backup' domain, because of possibility of overwriting # 'fake' volumes that have hardlinks with 'legal' volumes with same uuid and without hardlinks # First, check number of hardlinks ## volPath = os.path.join(cls.storage_repository, spUUID, sdUUID, sd.DOMAIN_IMAGES, imgUUID, pvolUUID) ## if os.path.exists(volPath): ## if os.stat(volPath).st_nlink == 1: ## return volList ## else: ## cls.log.info("Volume %s does not exist", volPath) ## return volList # scan whole domain pattern = os.path.join(repoPath, sdUUID, sd.DOMAIN_IMAGES, "*", "*.meta") files = oop.getProcessPool(sdUUID).glob.glob(pattern) for i in files: volid = os.path.splitext(os.path.basename(i))[0] imgUUID = os.path.basename(os.path.dirname(i)) if sdCache.produce(sdUUID).produceVolume(imgUUID, volid).getParent() == pvolUUID: volList.append({'imgUUID':imgUUID, 'volUUID':volid}) return volList
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, imgPath, volPath): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ sizeBytes = int(size) * BLOCK_SIZE try: oop.getProcessPool(dom.sdUUID).truncateFile(volPath, sizeBytes, creatExcl=True) except OSError as e: if e.errno == errno.EEXIST: raise se.VolumeAlreadyExists(volUUID) raise if preallocate == volume.PREALLOCATED_VOL: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting, sizeBytes) except ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info("Request to create %s volume %s with size = %s " "sectors", volume.type2name(volFormat), volPath, size) if volFormat == volume.COW_FORMAT: volume.createVolume(None, None, volPath, size, volFormat, preallocate) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(imgPath, volUUID, volFormat, preallocate) # By definition the volume is a leaf cls.file_setrw(volPath, rw=True) return (volPath,)
def create(cls, sdUUID, domainName, domClass, remotePath, storageType, version): """ Create new storage domain. 'sdUUID' - Storage Domain UUID 'domainName' - storage domain name ("iso" or "data domain name") 'domClass' - Data/Iso 'remotePath' - server:/export_path 'storageType' - NFS_DOMAIN, LOCALFS_DOMAIN, &etc. 'version' - DOMAIN_VERSIONS """ cls.log.info("sdUUID=%s domainName=%s remotePath=%s " "domClass=%s", sdUUID, domainName, remotePath, domClass) remotePath = fileUtils.normalize_path(remotePath) if not misc.isAscii(domainName) and not sd.supportsUnicode(version): raise se.UnicodeArgumentException() # Create local path mntPath = fileUtils.transformPath(remotePath) mntPoint = cls.getMountPoint(mntPath) cls._preCreateValidation(sdUUID, mntPoint, remotePath, storageType, version) domainDir = os.path.join(mntPoint, sdUUID) cls._prepareMetadata(domainDir, sdUUID, domainName, domClass, remotePath, storageType, version) # create domain images folder imagesDir = os.path.join(domainDir, sd.DOMAIN_IMAGES) oop.getProcessPool(sdUUID).fileUtils.createdir(imagesDir) # create special imageUUID for ISO/Floppy volumes if domClass is sd.ISO_DOMAIN: isoDir = os.path.join(imagesDir, sd.ISO_IMAGE_UUID) oop.getProcessPool(sdUUID).fileUtils.createdir(isoDir) fsd = cls(os.path.join(mntPoint, sdUUID)) fsd.initSPMlease() return fsd
def newVolumeLease(cls, sdUUID, volUUID, volPath): dom = sdCache.produce(sdUUID) procPool = oop.getProcessPool(sdUUID) if dom.hasVolumeLeases(): leasePath = cls.__leaseVolumePath(volPath) procPool.createSparseFile(leasePath, LEASE_FILEOFFSET) cls.file_setrw(leasePath, rw=True) sanlock.init_resource(sdUUID, volUUID, [(leasePath, LEASE_FILEOFFSET)])
def setMetadataPermissions(self): procPool = oop.getProcessPool(self.sdUUID) for metaFile in (sd.LEASES, sd.IDS, sd.INBOX, sd.OUTBOX): try: fpath = os.path.join(self.getMDPath(), metaFile) procPool.os.chmod(fpath, METADATA_PERMISSIONS) except Exception as e: raise se.StorageDomainMetadataCreationError( "Lease permission change file '%s' failed: %s" % (metaFile, e))
def setMetadataPermissions(self): procPool = oop.getProcessPool(self.sdUUID) for metaFile in (sd.LEASES, sd.IDS, sd.INBOX, sd.OUTBOX): try: fpath = os.path.join(self.getMDPath(), metaFile) procPool.os.chmod(fpath, 0660) except Exception as e: raise se.StorageDomainMetadataCreationError( "Lease permission change file '%s' failed: %s" % (metaFile, e))
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) procPool = oop.getProcessPool(sdUUID) procPool.fileUtils.createdir(metadataDir, 0o775) for metaFile, metaSize in FILE_SPECIAL_VOLUME_SIZES_MIB.iteritems(): try: procPool.truncateFile(os.path.join(metadataDir, metaFile), metaSize * constants.MEGAB, METADATA_PERMISSIONS) except Exception as e: raise se.StorageDomainMetadataCreationError( "create meta file '%s' failed: %s" % (metaFile, str(e))) metaFile = os.path.join(metadataDir, sd.METADATA) md = FileSDMetadata(metaFile) # initialize domain metadata content # FIXME : This is 99% like the metadata in block SD # Do we really need to keep the EXPORT_PATH? # no one uses it md.update({ sd.DMDK_VERSION: version, sd.DMDK_SDUUID: sdUUID, sd.DMDK_TYPE: storageType, sd.DMDK_CLASS: domClass, sd.DMDK_DESCRIPTION: domainName, sd.DMDK_ROLE: sd.REGULAR_DOMAIN, sd.DMDK_POOLS: [], sd.DMDK_LOCK_POLICY: '', sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_LEASE_TIME_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_TIME_SEC], sd.DMDK_IO_OP_TIMEOUT_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_IO_OP_TIMEOUT_SEC], sd.DMDK_LEASE_RETRIES: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_RETRIES], REMOTE_PATH: remotePath })
def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate, volParent, srcImgUUID, srcVolUUID, imgPath, volPath): """ Class specific implementation of volumeCreate. All the exceptions are properly handled and logged in volume.create() """ sizeBytes = int(size) * BLOCK_SIZE if preallocate == volume.SPARSE_VOL: # Sparse = regular file oop.getProcessPool(dom.sdUUID).truncateFile(volPath, sizeBytes) else: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting, sizeBytes) except ActionStopped: raise except Exception: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(volPath) if not volParent: cls.log.info( "Request to create %s volume %s with size = %s " "sectors", volume.type2name(volFormat), volPath, size) if volFormat == volume.COW_FORMAT: volume.createVolume(None, None, volPath, size, volFormat, preallocate) else: # Create hardlink to template and its meta file cls.log.info("Request to create snapshot %s/%s of volume %s/%s", imgUUID, volUUID, srcImgUUID, srcVolUUID) volParent.clone(imgPath, volUUID, volFormat, preallocate) # By definition the volume is a leaf cls.file_setrw(volPath, rw=True) return (volPath, )
def getImageVolumes(cls, repoPath, sdUUID, imgUUID): """ Fetch the list of the Volumes UUIDs, not including the shared base (template) """ # Get Volumes of an image pattern = os.path.join(os.path.join(repoPath, sdUUID, sd.DOMAIN_IMAGES, imgUUID, "*.meta")) files = oop.getProcessPool(sdUUID).glob.glob(pattern) volList = [] for i in files: volid = os.path.splitext(os.path.basename(i))[0] if sdCache.produce(sdUUID).produceVolume(imgUUID, volid).getImage() == imgUUID: volList.append(volid) return volList
def create(cls, repoPath, sdUUID, imgUUID, size, volFormat, preallocate, diskType, volUUID, desc, srcImgUUID, srcVolUUID): """ Create a new volume with given size or snapshot 'size' - in sectors 'volFormat' - volume format COW / RAW 'preallocate' - Prealocate / Sparse 'diskType' - string that describes disk type System|Data|Shared|Swap|Temp 'srcImgUUID' - source image UUID 'srcVolUUID' - source volume UUID """ if not volUUID: volUUID = str(uuid.uuid4()) if volUUID == volume.BLANK_UUID: raise se.InvalidParameterException("volUUID", volUUID) # Validate volume parameters should be checked here for all # internal flows using volume creation. cls.validateCreateVolumeParams(volFormat, preallocate, srcVolUUID) imageDir = image.Image(repoPath).create(sdUUID, imgUUID) vol_path = os.path.join(imageDir, volUUID) voltype = "LEAF" pvol = None # Check if volume already exists if oop.getProcessPool(sdUUID).fileUtils.pathExists(vol_path): raise se.VolumeAlreadyExists(vol_path) # Check if snapshot creation required if srcVolUUID != volume.BLANK_UUID: if srcImgUUID == volume.BLANK_UUID: srcImgUUID = imgUUID pvol = FileVolume(repoPath, sdUUID, srcImgUUID, srcVolUUID) # Cannot create snapshot for ILLEGAL volume if not pvol.isLegal(): raise se.createIllegalVolumeSnapshotError(pvol.volUUID) # create volume rollback vars.task.pushRecovery( task.Recovery("halfbaked volume rollback", "fileVolume", "FileVolume", "halfbakedVolumeRollback", [vol_path])) if preallocate == volume.PREALLOCATED_VOL: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", vol_path, vars.task.aborting, (int(size) * 512)) except se.ActionStopped, e: raise e except Exception, e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(vol_path)
def create(cls, repoPath, sdUUID, imgUUID, size, volFormat, preallocate, diskType, volUUID, desc, srcImgUUID, srcVolUUID): """ Create a new volume with given size or snapshot 'size' - in sectors 'volFormat' - volume format COW / RAW 'preallocate' - Prealocate / Sparse 'diskType' - string that describes disk type System|Data|Shared|Swap|Temp 'srcImgUUID' - source image UUID 'srcVolUUID' - source volume UUID """ if not volUUID: volUUID = str(uuid.uuid4()) if volUUID == volume.BLANK_UUID: raise se.InvalidParameterException("volUUID", volUUID) # Validate volume parameters should be checked here for all # internal flows using volume creation. cls.validateCreateVolumeParams(volFormat, preallocate, srcVolUUID) imageDir = image.Image(repoPath).create(sdUUID, imgUUID) vol_path = os.path.join(imageDir, volUUID) voltype = "LEAF" pvol = None # Check if volume already exists if oop.getProcessPool(sdUUID).fileUtils.pathExists(vol_path): raise se.VolumeAlreadyExists(vol_path) # Check if snapshot creation required if srcVolUUID != volume.BLANK_UUID: if srcImgUUID == volume.BLANK_UUID: srcImgUUID = imgUUID pvol = FileVolume(repoPath, sdUUID, srcImgUUID, srcVolUUID) # Cannot create snapshot for ILLEGAL volume if not pvol.isLegal(): raise se.createIllegalVolumeSnapshotError(pvol.volUUID) # Rollback sentinel, just to mark the start of the task vars.task.pushRecovery(task.Recovery(task.ROLLBACK_SENTINEL, "fileVolume", "FileVolume", "startCreateVolumeRollback", [sdUUID, imgUUID, volUUID])) # create volume rollback vars.task.pushRecovery(task.Recovery("halfbaked volume rollback", "fileVolume", "FileVolume", "halfbakedVolumeRollback", [vol_path])) if preallocate == volume.PREALLOCATED_VOL: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", vol_path, vars.task.aborting, (int(size) * 512)) except se.ActionStopped, e: raise e except Exception, e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(vol_path)
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) procPool = oop.getProcessPool(sdUUID) procPool.fileUtils.createdir(metadataDir, 0775) for metaFile in (sd.LEASES, sd.IDS, sd.INBOX, sd.OUTBOX): try: procPool.createSparseFile(os.path.join(metadataDir, metaFile), 0, 0660) except Exception, e: raise se.StorageDomainMetadataCreationError("create meta file '%s' failed: %s" % (metaFile, str(e)))
def create(cls, sdUUID, domainName, domClass, remotePath, storageType, version): """ Create new storage domain. 'sdUUID' - Storage Domain UUID 'domainName' - storage domain name ("iso" or "data domain name") 'remotePath' - server:/export_path 'domClass' - Data/Iso """ cls.log.info("sdUUID=%s domainName=%s remotePath=%s " "domClass=%s", sdUUID, domainName, remotePath, domClass) # Create local path mntPath = fileUtils.transformPath(remotePath) mntPoint = os.path.join(cls.storage_repository, sd.DOMAIN_MNT_POINT, mntPath) cls._preCreateValidation(sdUUID, mntPoint, remotePath, version) domainDir = os.path.join(mntPoint, sdUUID) cls._prepareMetadata(domainDir, sdUUID, domainName, domClass, remotePath, storageType, version) # create domain images folder imagesDir = os.path.join(domainDir, sd.DOMAIN_IMAGES) oop.getProcessPool(sdUUID).fileUtils.createdir(imagesDir) # create special imageUUID for ISO/Floppy volumes if domClass is sd.ISO_DOMAIN: isoDir = os.path.join(imagesDir, sd.ISO_IMAGE_UUID) oop.getProcessPool(sdUUID).fileUtils.createdir(isoDir) fsd = NfsStorageDomain(os.path.join(mntPoint, sdUUID)) fsd.initSPMlease() return fsd
def getImageVolumes(cls, repoPath, sdUUID, imgUUID): """ Fetch the list of the Volumes UUIDs, not including the shared base (template) """ # Get Volumes of an image pattern = os.path.join(repoPath, sdUUID, sd.DOMAIN_IMAGES, imgUUID, "*.meta") files = oop.getProcessPool(sdUUID).glob.glob(pattern) volList = [] for i in files: volid = os.path.splitext(os.path.basename(i))[0] if (sdCache.produce(sdUUID).produceVolume( imgUUID, volid).getImage() == imgUUID): volList.append(volid) return volList
def collectMetaFiles(possibleDomain): try: metaFiles = oop.getProcessPool(possibleDomain).glob.glob( os.path.join(possibleDomain, constants.UUID_GLOB_PATTERN, sd.DOMAIN_META_DATA) ) for metaFile in metaFiles: if os.path.basename(os.path.dirname(metaFile)) != sd.MASTER_FS_DIR: sdUUID = os.path.basename(os.path.dirname(metaFile)) return (sdUUID, os.path.dirname(metaFile)) except Timeout: log.warn("Metadata collection for domain path %s timedout", possibleDomain, exc_info=True) except Exception: log.warn("Could not collect metadata file for domain path %s", possibleDomain, exc_info=True)
def _preCreateValidation(cls, sdUUID, domPath, typeSpecificArg, version): # Some trivial resource validation if ":" not in typeSpecificArg: raise se.StorageDomainIllegalRemotePath(typeSpecificArg) sd.validateDomainVersion(version) # Make sure the underlying file system is mounted if not fileUtils.isMounted(mountPoint=domPath, mountType=fileUtils.FSTYPE_NFS): raise se.StorageDomainFSNotMounted(typeSpecificArg) validateDirAccess(domPath) # Make sure there are no remnants of other domain mdpat = os.path.join(domPath, "*", sd.DOMAIN_META_DATA) if len(oop.getProcessPool(sdUUID).glob.glob(mdpat)) > 0: raise se.StorageDomainNotEmpty(typeSpecificArg)
def _prepareMetadata(cls, domPath, sdUUID, domainName, domClass, remotePath, storageType, version): """ Prepare all domain's special volumes and metadata """ # create domain metadata folder metadataDir = os.path.join(domPath, sd.DOMAIN_META_DATA) procPool = oop.getProcessPool(sdUUID) procPool.fileUtils.createdir(metadataDir, 0o775) for metaFile, metaSize in FILE_SPECIAL_VOLUME_SIZES_MIB.iteritems(): try: procPool.truncateFile( os.path.join(metadataDir, metaFile), metaSize * constants.MEGAB, METADATA_PERMISSIONS) except Exception as e: raise se.StorageDomainMetadataCreationError( "create meta file '%s' failed: %s" % (metaFile, str(e))) metaFile = os.path.join(metadataDir, sd.METADATA) md = FileSDMetadata(metaFile) # initialize domain metadata content # FIXME : This is 99% like the metadata in block SD # Do we really need to keep the EXPORT_PATH? # no one uses it md.update({ sd.DMDK_VERSION: version, sd.DMDK_SDUUID: sdUUID, sd.DMDK_TYPE: storageType, sd.DMDK_CLASS: domClass, sd.DMDK_DESCRIPTION: domainName, sd.DMDK_ROLE: sd.REGULAR_DOMAIN, sd.DMDK_POOLS: [], sd.DMDK_LOCK_POLICY: '', sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LOCK_RENEWAL_INTERVAL_SEC], sd.DMDK_LEASE_TIME_SEC: sd.DEFAULT_LEASE_PARAMS[ sd.DMDK_LEASE_TIME_SEC], sd.DMDK_IO_OP_TIMEOUT_SEC: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_IO_OP_TIMEOUT_SEC], sd.DMDK_LEASE_RETRIES: sd.DEFAULT_LEASE_PARAMS[sd.DMDK_LEASE_RETRIES], REMOTE_PATH: remotePath })
def collectMetaFiles(possibleDomain): try: metaFiles = oop.getProcessPool(possibleDomain).glob.glob( os.path.join(possibleDomain, constants.UUID_GLOB_PATTERN, sd.DOMAIN_META_DATA)) for metaFile in metaFiles: if (os.path.basename(os.path.dirname(metaFile)) != sd.MASTER_FS_DIR): sdUUID = os.path.basename(os.path.dirname(metaFile)) return (sdUUID, os.path.dirname(metaFile)) except Exception: log.warn("Could not collect metadata file for domain path %s", possibleDomain, exc_info=True)
def getChildren(self): """ Return children volume UUIDs. Children can be found in any image of the volume SD. """ domPath = self.imagePath.split('images')[0] metaPattern = os.path.join(domPath, 'images', '*', '*.meta') metaPaths = oop.getProcessPool(self.sdUUID).glob.glob(metaPattern) pattern = "%s.*%s" % (volume.PUUID, self.volUUID) matches = grepCmd(pattern, metaPaths) if matches: children = [] for line in matches: volMeta = os.path.basename(line.split(':')[0]) children.append(os.path.splitext(volMeta)[0]) # volUUID else: children = tuple() return tuple(children)
def getChildren(self): """ Return children volume UUIDs. This API is not suitable for use with a template's base volume. """ domPath = self.imagePath.split('images')[0] metaPattern = os.path.join(domPath, 'images', self.imgUUID, '*.meta') metaPaths = oop.getProcessPool(self.sdUUID).glob.glob(metaPattern) pattern = "%s.*%s" % (sc.PUUID, self.volUUID) matches = grepCmd(pattern, metaPaths) if matches: children = [] for line in matches: volMeta = os.path.basename(line.rsplit(':', 1)[0]) children.append(os.path.splitext(volMeta)[0]) # volUUID else: children = tuple() return tuple(children)
def getChildren(self): """ Return children volume UUIDs. This API is not suitable for use with a template's base volume. """ domPath = self.imagePath.split('images')[0] metaPattern = os.path.join(domPath, 'images', self.imgUUID, '*.meta') metaPaths = oop.getProcessPool(self.sdUUID).glob.glob(metaPattern) pattern = "%s.*%s" % (volume.PUUID, self.volUUID) matches = grepCmd(pattern, metaPaths) if matches: children = [] for line in matches: volMeta = os.path.basename(line.rsplit(':', 1)[0]) children.append(os.path.splitext(volMeta)[0]) # volUUID else: children = tuple() return tuple(children)
def _preCreateValidation(cls, sdUUID, domPath, typeSpecificArg, storageType, version): # Some trivial resource validation # TODO Checking storageType==nfs in the nfs class is not clean if storageType == sd.NFS_DOMAIN and ":" not in typeSpecificArg: raise se.StorageDomainIllegalRemotePath(typeSpecificArg) sd.validateDomainVersion(version) # Make sure the underlying file system is mounted if not mount.isMounted(domPath): raise se.StorageDomainFSNotMounted(domPath) fileSD.validateDirAccess(domPath) fileSD.validateFileSystemFeatures(sdUUID, domPath) # Make sure there are no remnants of other domain mdpat = os.path.join(domPath, "*", sd.DOMAIN_META_DATA) if len(oop.getProcessPool(sdUUID).glob.glob(mdpat)) > 0: raise se.StorageDomainNotEmpty(typeSpecificArg)
def shareVolumeRollback(cls, taskObj, volPath): cls.log.info("Volume rollback for volPath=%s", volPath) procPool = oop.getProcessPool(getDomUuidFromVolumePath(volPath)) procPool.utils.rmFile(volPath) procPool.utils.rmFile(cls.manifestClass.metaVolumePath(volPath)) procPool.utils.rmFile(cls.manifestClass.leaseVolumePath(volPath))
def __init__(self, metafile): # FileSDMetadata is kept in the file self._metafile = metafile self._sdUUID = getDomUuidFromMetafilePath(metafile) self._oop = oop.getProcessPool(self._sdUUID)
def oop(self): return oop.getProcessPool(self.sdUUID)
def getReadDelay(self): t = time.time() oop.getProcessPool(self.sdUUID).directReadLines(self.metafile) return time.time() - t
vars.task.pushRecovery( task.Recovery("halfbaked volume rollback", "fileVolume", "FileVolume", "halfbakedVolumeRollback", [vol_path])) if preallocate == volume.PREALLOCATED_VOL: try: # ddWatchCopy expects size to be in bytes misc.ddWatchCopy("/dev/zero", vol_path, vars.task.aborting, (int(size) * 512)) except se.ActionStopped, e: raise e except Exception, e: cls.log.error("Unexpected error", exc_info=True) raise se.VolumesZeroingError(vol_path) else: # Sparse = Normal file oop.getProcessPool(sdUUID).createSparseFile(vol_path, 0) cls.log.info("fileVolume: create: volUUID %s srcImg %s srvVol %s" % (volUUID, srcImgUUID, srcVolUUID)) if not pvol: cls.log.info( "Request to create %s volume %s with size = %s sectors", volume.type2name(volFormat), vol_path, size) # Create preallocate/raw volume via qemu-img actually redundant if preallocate == volume.SPARSE_VOL or volFormat == volume.COW_FORMAT: volume.createVolume(None, None, vol_path, size, volFormat, preallocate) else: # Create hardlink to template and its meta file if imgUUID != srcImgUUID: pvol.share(imageDir, hard=True)
def shareVolumeRollback(cls, taskObj, volPath): cls.log.info("Volume rollback for volPath=%s", volPath) procPool = oop.getProcessPool(getDomUuidFromVolumePath(volPath)) procPool.utils.rmFile(volPath) procPool.utils.rmFile(cls.__metaVolumePath(volPath)) procPool.utils.rmFile(cls.__leaseVolumePath(volPath))
def halfbakedVolumeRollback(cls, taskObj, volPath): cls.log.info("halfbakedVolumeRollback: volPath=%s" % (volPath)) sdUUID = getDomUuidFromVolumePath(volPath) if oop.getProcessPool(sdUUID).fileUtils.pathExists(volPath): oop.getProcessPool(sdUUID).os.unlink(volPath)
def createVolumeMetadataRollback(cls, taskObj, volPath): cls.log.info("createVolumeMetadataRollback: volPath=%s" % (volPath)) metaPath = cls.manifestClass.metaVolumePath(volPath) sdUUID = getDomUuidFromVolumePath(volPath) if oop.getProcessPool(sdUUID).os.path.lexists(metaPath): oop.getProcessPool(sdUUID).os.unlink(metaPath)