def test_convert_loop(nb, srcUnit, dstUnit): """ Make sure that converting a size back and forth preserves the number """ converted = convertSizeUnits(convertSizeUnits(nb, srcUnit, dstUnit), dstUnit, srcUnit) # We exclude the infinity case if converted != float('Inf'): assert converted == nb
def test_convert_loop(nb, srcUnit, dstUnit): """Make sure that converting a size back and forth preserves the number""" converted = convertSizeUnits(convertSizeUnits(nb, srcUnit, dstUnit), dstUnit, srcUnit) # We exclude the infinity case if converted != float("Inf"): assert converted == nb
def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. It will return a dictionary with th results, converted to "correct" unit. """ params = self._prepareCommand() if not params["OK"]: return params elementName, unit = params["Value"] result = self.rmClient.selectSpaceTokenOccupancyCache( token=elementName) if not result["OK"]: return result if not result["Value"]: return S_ERROR(errno.ENODATA, "No occupancy recorded") # results are normally in 'MB' free = result["Value"][0][3] total = result["Value"][0][4] free = convertSizeUnits(free, "MB", unit) total = convertSizeUnits(total, "MB", unit) if free == -sys.maxsize or total == -sys.maxsize: return S_ERROR("No valid unit specified") return S_OK({"Free": free, "Total": total})
def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. It will return a dictionary with th results, converted to "correct" unit. """ params = self._prepareCommand() if not params['OK']: return params elementName, unit = params['Value'] result = self.rmClient.selectSpaceTokenOccupancyCache( token=elementName) if not result['OK']: return result # results are normally in 'MB' free = result['Value'][0][3] total = result['Value'][0][4] free = convertSizeUnits(free, 'MB', unit) total = convertSizeUnits(total, 'MB', unit) if free == -sys.maxsize or total == -sys.maxsize: return S_ERROR("No valid unit specified") return S_OK({'Free': free, 'Total': total})
def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. It will return a dictionary with th results, converted to "correct" unit. """ params = self._prepareCommand() if not params['OK']: return params elementName, unit = params['Value'] result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) if not result['OK']: return result if not result['Value']: return S_ERROR(errno.ENODATA, "No occupancy recorded") # results are normally in 'MB' free = result['Value'][0][3] total = result['Value'][0][4] free = convertSizeUnits(free, 'MB', unit) total = convertSizeUnits(total, 'MB', unit) if free == -sys.maxsize or total == -sys.maxsize: return S_ERROR("No valid unit specified") return S_OK({'Free': free, 'Total': total})
def test_convert_to_bigger_unit_floats(nb): """ Make sure that converting to bigger unit gets the number smaller . Also tests that two steps are equal to two consecutive steps """ toKB = convertSizeUnits(nb, 'B', 'kB') toMB = convertSizeUnits(nb, 'B', 'MB') fromkBtoMB = convertSizeUnits(toKB, 'kB', 'MB') assert toKB < nb assert toMB < toKB assert toMB == fromkBtoMB
def export_getAdminInfo(): """ Send the storage element administration information """ storageDict = {} storageDict['BasePath'] = BASE_PATH storageDict['MaxCapacity'] = MAX_STORAGE_SIZE used_space = getDirectorySize(BASE_PATH) stats = os.statvfs(BASE_PATH) available_space = convertSizeUnits(stats.f_bsize * stats.f_bavail, 'B', 'MB') allowed_space = convertSizeUnits(MAX_STORAGE_SIZE, 'B', 'MB') - used_space actual_space = min(available_space, allowed_space) storageDict['AvailableSpace'] = actual_space storageDict['UsedSpace'] = used_space return S_OK(storageDict)
def getOccupancy(self, unit='MB', **kwargs): """ Retrieves the space information about the storage. It returns the Total, Guaranteed and Free space . It loops over the different Storage Plugins to query it. :returns: S_OK with dict (keys: Total, Guaranteed, Free) """ log = self.log.getSubLogger('getOccupancy', True) filteredPlugins = self.__filterPlugins('getOccupancy') if not filteredPlugins: return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy") # Try all of the storages one by one for storage in filteredPlugins: # The result of the plugin is always in MB res = storage.getOccupancy(**kwargs) if res['OK']: occupancyDict = res['Value'] if unit != 'MB': for space in ['Total', 'Free']: convertedSpace = convertSizeUnits(occupancyDict[space], 'MB', unit) # If we have a conversion error, we go to the next plugin if convertedSpace == -sys.maxsize: log.verbose( "Error converting %s space from MB to %s: %s" % (space, unit, occupancyDict[space])) break occupancyDict[space] = convertedSpace return res return S_ERROR("Could not retrieve the occupancy from any plugin")
def getOccupancy(self, unit='MB', **kwargs): """ Retrieves the space information about the storage. It returns the Total and Free space. It loops over the different Storage Plugins to query it. :params occupancyLFN: (named param) LFN where to find the space reporting json file on the storage The json file should contain the Free and Total space in MB. If not specified, the default path will be </vo/occupancy.json> :returns: S_OK with dict (keys: Total, Free) """ log = self.log.getSubLogger('getOccupancy', True) # Mandatory parameters mandatoryParams = set(['Total', 'Free']) if 'occupancyLFN' not in kwargs: occupancyLFN = self.options.get('OccupancyLFN') if not occupancyLFN: occupancyLFN = os.path.join('/', self.vo, DEFAULT_OCCUPANCY_FILE) kwargs['occupancyLFN'] = occupancyLFN filteredPlugins = self.__filterPlugins('getOccupancy') if not filteredPlugins: return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy") # Try all of the storages one by one for storage in filteredPlugins: # The result of the plugin is always in MB res = storage.getOccupancy(**kwargs) if res['OK']: occupancyDict = res['Value'] # Make sure all the mandatory parameters are present if set(occupancyDict) & mandatoryParams != mandatoryParams: log.verbose("Missing mandatory parameters", mandatoryParams - set(occupancyDict)) continue if unit != 'MB': for space in ['Total', 'Free']: convertedSpace = convertSizeUnits(occupancyDict[space], 'MB', unit) # If we have a conversion error, we go to the next plugin if convertedSpace == -sys.maxsize: log.verbose( "Error converting %s space from MB to %s: %s" % (space, unit, occupancyDict[space])) break occupancyDict[space] = convertedSpace return res return S_ERROR("Could not retrieve the occupancy from any plugin")
def _storeCommand(self, results): """ Stores the results in the cache (SpaceTokenOccupancyCache), and adds records to the StorageOccupancy accounting. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0, 'SpaceReservation': 'LHCb-Disk'} :returns: S_OK/S_ERROR dict """ # Stores in cache res = self.rmClient.addOrModifySpaceTokenOccupancyCache( endpoint=results['Endpoint'], lastCheckTime=datetime.utcnow(), free=results['Free'], total=results['Total'], token=results['ElementName']) if not res['OK']: self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res['Message']) return res # Now proceed with the accounting siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName']) if not siteRes['OK']: return siteRes accountingDict = { 'StorageElement': results['ElementName'], 'Endpoint': results['Endpoint'], 'Site': siteRes['Value'] if siteRes['Value'] else 'unassigned' } results['Used'] = results['Total'] - results['Free'] for sType in ['Total', 'Free', 'Used']: spaceTokenAccounting = StorageOccupancy() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey('SpaceType', sType) spaceTokenAccounting.setValueByKey( 'Space', int(convertSizeUnits(results[sType], 'MB', 'B'))) res = gDataStoreClient.addRegister(spaceTokenAccounting) if not res['OK']: self.log.warn("Could not commit register", res['Message']) continue return gDataStoreClient.commit()
def _storeCommand(self, results): """ Stores the results in the cache (SpaceTokenOccupancyCache), and adds records to the StorageOccupancy accounting. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0} :returns: S_OK/S_ERROR dict """ # Stores in cache res = self.rmClient.addOrModifySpaceTokenOccupancyCache( endpoint=results["Endpoint"], lastCheckTime=datetime.utcnow(), free=results["Free"], total=results["Total"], token=results["ElementName"], ) if not res["OK"]: self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res["Message"]) return res # Now proceed with the accounting siteRes = DMSHelpers().getLocalSiteForSE(results["ElementName"]) if not siteRes["OK"]: return siteRes accountingDict = { "StorageElement": results["ElementName"], "Endpoint": results["Endpoint"], "Site": siteRes["Value"] if siteRes["Value"] else "unassigned", } results["Used"] = results["Total"] - results["Free"] for sType in ["Total", "Free", "Used"]: spaceTokenAccounting = StorageOccupancy() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey("SpaceType", sType) spaceTokenAccounting.setValueByKey( "Space", int(convertSizeUnits(results[sType], "MB", "B"))) res = gDataStoreClient.addRegister(spaceTokenAccounting) if not res["OK"]: self.log.warn("Could not commit register", res["Message"]) continue return gDataStoreClient.commit()
def _storeCommand(self, results): """ _storeCommand Adding records to accounting, on top of what does the derived method. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0, 'SpaceReservation': 'LHCb-Disk'} :returns: S_OK/S_ERROR dict """ res = super(FreeDiskSpaceCommand, self)._storeCommand(results) if not res['OK']: return res siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName']) if not siteRes['OK']: return siteRes if not siteRes['Value']: return S_OK() spaceReservation = results.get('SpaceReservation') accountingDict = { 'SpaceToken': spaceReservation, 'Endpoint': results['Endpoint'], 'Site': siteRes['Value'] } results['Used'] = results['Total'] - results['Free'] for sType in ['Total', 'Free', 'Used']: spaceTokenAccounting = SpaceToken() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey('SpaceType', sType) spaceTokenAccounting.setValueByKey( 'Space', int(convertSizeUnits(results[sType], 'MB', 'B'))) gDataStoreClient.addRegister(spaceTokenAccounting) gDataStoreClient.commit() return S_OK()
def initializeStorageElementHandler(serviceInfo): """ Initialize Storage Element global settings """ global BASE_PATH global USE_TOKENS global MAX_STORAGE_SIZE BASE_PATH = getServiceOption(serviceInfo, "BasePath", BASE_PATH) if not BASE_PATH: gLogger.error('Failed to get the base path') return S_ERROR('Failed to get the base path') mkDir(BASE_PATH) USE_TOKENS = getServiceOption(serviceInfo, "%UseTokens", USE_TOKENS) MAX_STORAGE_SIZE = convertSizeUnits(getServiceOption(serviceInfo, "MaxStorageSize", MAX_STORAGE_SIZE), 'MB', 'B') gLogger.info('Starting DIRAC Storage Element') gLogger.info('Base Path: %s' % BASE_PATH) gLogger.info('Max size: %d Bytes' % MAX_STORAGE_SIZE) gLogger.info('Use access control tokens: ' + str(USE_TOKENS)) return S_OK()
def initializeStorageElementHandler(serviceInfo): """Initialize Storage Element global settings""" global BASE_PATH global USE_TOKENS global MAX_STORAGE_SIZE BASE_PATH = getServiceOption(serviceInfo, "BasePath", "") if not BASE_PATH: gLogger.error("Failed to get the base path") return S_ERROR("Failed to get the base path") mkDir(BASE_PATH) USE_TOKENS = getServiceOption(serviceInfo, "UseTokens", USE_TOKENS) MAX_STORAGE_SIZE = convertSizeUnits( getServiceOption(serviceInfo, "MaxStorageSize", MAX_STORAGE_SIZE), "MB", "B") gLogger.info("Starting DIRAC Storage Element") gLogger.info("Base Path: %s" % BASE_PATH) gLogger.info("Max size: %d Bytes" % MAX_STORAGE_SIZE) gLogger.info("Use access control tokens: " + str(USE_TOKENS)) return S_OK()
def test_convert_error_to_maxint(): """ Make sure that on error we receive -sys.maxint """ assert convertSizeUnits('size', 'B', 'kB') == -sys.maxsize assert convertSizeUnits(0, 'srcUnit', 'kB') == -sys.maxsize assert convertSizeUnits(0, 'B', 'dstUnit') == -sys.maxsize
def test_convert_error_to_maxint(): """Make sure that on error we receive -sys.maxint""" assert convertSizeUnits("size", "B", "kB") == -sys.maxsize assert convertSizeUnits(0, "srcUnit", "kB") == -sys.maxsize assert convertSizeUnits(0, "B", "dstUnit") == -sys.maxsize