def test_convert_loop(nb, srcUnit, dstUnit):
""" Make sure that converting a size back and forth preserves the number """
converted = convertSizeUnits(convertSizeUnits(nb, srcUnit, dstUnit), dstUnit, srcUnit)
# We exclude the infinity case
if converted != float('Inf'):
assert converted == nb
def test_convert_loop(nb, srcUnit, dstUnit):
"""Make sure that converting a size back and forth preserves the number"""
converted = convertSizeUnits(convertSizeUnits(nb, srcUnit, dstUnit), dstUnit, srcUnit)
# We exclude the infinity case
if converted != float("Inf"):
assert converted == nb
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
It will return a dictionary with th results, converted to "correct" unit.
"""
params = self._prepareCommand()
if not params["OK"]:
return params
elementName, unit = params["Value"]
result = self.rmClient.selectSpaceTokenOccupancyCache(
token=elementName)
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR(errno.ENODATA, "No occupancy recorded")
# results are normally in 'MB'
free = result["Value"][0][3]
total = result["Value"][0][4]
free = convertSizeUnits(free, "MB", unit)
total = convertSizeUnits(total, "MB", unit)
if free == -sys.maxsize or total == -sys.maxsize:
return S_ERROR("No valid unit specified")
return S_OK({"Free": free, "Total": total})
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
It will return a dictionary with th results, converted to "correct" unit.
"""
params = self._prepareCommand()
if not params['OK']:
return params
elementName, unit = params['Value']
result = self.rmClient.selectSpaceTokenOccupancyCache(
token=elementName)
if not result['OK']:
return result
# results are normally in 'MB'
free = result['Value'][0][3]
total = result['Value'][0][4]
free = convertSizeUnits(free, 'MB', unit)
total = convertSizeUnits(total, 'MB', unit)
if free == -sys.maxsize or total == -sys.maxsize:
return S_ERROR("No valid unit specified")
return S_OK({'Free': free, 'Total': total})
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
It will return a dictionary with th results, converted to "correct" unit.
"""
params = self._prepareCommand()
if not params['OK']:
return params
elementName, unit = params['Value']
result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName)
if not result['OK']:
return result
if not result['Value']:
return S_ERROR(errno.ENODATA, "No occupancy recorded")
# results are normally in 'MB'
free = result['Value'][0][3]
total = result['Value'][0][4]
free = convertSizeUnits(free, 'MB', unit)
total = convertSizeUnits(total, 'MB', unit)
if free == -sys.maxsize or total == -sys.maxsize:
return S_ERROR("No valid unit specified")
return S_OK({'Free': free, 'Total': total})
def test_convert_to_bigger_unit_floats(nb):
""" Make sure that converting to bigger unit gets the number smaller .
Also tests that two steps are equal to two consecutive steps
"""
toKB = convertSizeUnits(nb, 'B', 'kB')
toMB = convertSizeUnits(nb, 'B', 'MB')
fromkBtoMB = convertSizeUnits(toKB, 'kB', 'MB')
assert toKB < nb
assert toMB < toKB
assert toMB == fromkBtoMB
def test_convert_to_bigger_unit_floats(nb):
""" Make sure that converting to bigger unit gets the number smaller .
Also tests that two steps are equal to two consecutive steps
"""
toKB = convertSizeUnits(nb, 'B', 'kB')
toMB = convertSizeUnits(nb, 'B', 'MB')
fromkBtoMB = convertSizeUnits(toKB, 'kB', 'MB')
assert toKB < nb
assert toMB < toKB
assert toMB == fromkBtoMB
def export_getAdminInfo():
""" Send the storage element administration information
"""
storageDict = {}
storageDict['BasePath'] = BASE_PATH
storageDict['MaxCapacity'] = MAX_STORAGE_SIZE
used_space = getDirectorySize(BASE_PATH)
stats = os.statvfs(BASE_PATH)
available_space = convertSizeUnits(stats.f_bsize * stats.f_bavail, 'B', 'MB')
allowed_space = convertSizeUnits(MAX_STORAGE_SIZE, 'B', 'MB') - used_space
actual_space = min(available_space, allowed_space)
storageDict['AvailableSpace'] = actual_space
storageDict['UsedSpace'] = used_space
return S_OK(storageDict)
def getOccupancy(self, unit='MB', **kwargs):
""" Retrieves the space information about the storage.
It returns the Total, Guaranteed and Free space .
It loops over the different Storage Plugins to query it.
:returns: S_OK with dict (keys: Total, Guaranteed, Free)
"""
log = self.log.getSubLogger('getOccupancy', True)
filteredPlugins = self.__filterPlugins('getOccupancy')
if not filteredPlugins:
return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy")
# Try all of the storages one by one
for storage in filteredPlugins:
# The result of the plugin is always in MB
res = storage.getOccupancy(**kwargs)
if res['OK']:
occupancyDict = res['Value']
if unit != 'MB':
for space in ['Total', 'Free']:
convertedSpace = convertSizeUnits(occupancyDict[space], 'MB', unit)
# If we have a conversion error, we go to the next plugin
if convertedSpace == -sys.maxsize:
log.verbose(
"Error converting %s space from MB to %s: %s" %
(space, unit, occupancyDict[space]))
break
occupancyDict[space] = convertedSpace
return res
return S_ERROR("Could not retrieve the occupancy from any plugin")
def getOccupancy(self, unit='MB', **kwargs):
""" Retrieves the space information about the storage.
It returns the Total and Free space.
It loops over the different Storage Plugins to query it.
:params occupancyLFN: (named param) LFN where to find the space reporting json file on the storage
The json file should contain the Free and Total space in MB.
If not specified, the default path will be </vo/occupancy.json>
:returns: S_OK with dict (keys: Total, Free)
"""
log = self.log.getSubLogger('getOccupancy', True)
# Mandatory parameters
mandatoryParams = set(['Total', 'Free'])
if 'occupancyLFN' not in kwargs:
occupancyLFN = self.options.get('OccupancyLFN')
if not occupancyLFN:
occupancyLFN = os.path.join('/', self.vo, DEFAULT_OCCUPANCY_FILE)
kwargs['occupancyLFN'] = occupancyLFN
filteredPlugins = self.__filterPlugins('getOccupancy')
if not filteredPlugins:
return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy")
# Try all of the storages one by one
for storage in filteredPlugins:
# The result of the plugin is always in MB
res = storage.getOccupancy(**kwargs)
if res['OK']:
occupancyDict = res['Value']
# Make sure all the mandatory parameters are present
if set(occupancyDict) & mandatoryParams != mandatoryParams:
log.verbose("Missing mandatory parameters", mandatoryParams - set(occupancyDict))
continue
if unit != 'MB':
for space in ['Total', 'Free']:
convertedSpace = convertSizeUnits(occupancyDict[space], 'MB', unit)
# If we have a conversion error, we go to the next plugin
if convertedSpace == -sys.maxsize:
log.verbose(
"Error converting %s space from MB to %s: %s" %
(space, unit, occupancyDict[space]))
break
occupancyDict[space] = convertedSpace
return res
return S_ERROR("Could not retrieve the occupancy from any plugin")
def _storeCommand(self, results):
"""
Stores the results in the cache (SpaceTokenOccupancyCache),
and adds records to the StorageOccupancy accounting.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0,
'SpaceReservation': 'LHCb-Disk'}
:returns: S_OK/S_ERROR dict
"""
# Stores in cache
res = self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results['Endpoint'],
lastCheckTime=datetime.utcnow(),
free=results['Free'],
total=results['Total'],
token=results['ElementName'])
if not res['OK']:
self.log.error("Error calling addOrModifySpaceTokenOccupancyCache",
res['Message'])
return res
# Now proceed with the accounting
siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName'])
if not siteRes['OK']:
return siteRes
accountingDict = {
'StorageElement': results['ElementName'],
'Endpoint': results['Endpoint'],
'Site': siteRes['Value'] if siteRes['Value'] else 'unassigned'
}
results['Used'] = results['Total'] - results['Free']
for sType in ['Total', 'Free', 'Used']:
spaceTokenAccounting = StorageOccupancy()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey('SpaceType', sType)
spaceTokenAccounting.setValueByKey(
'Space', int(convertSizeUnits(results[sType], 'MB', 'B')))
res = gDataStoreClient.addRegister(spaceTokenAccounting)
if not res['OK']:
self.log.warn("Could not commit register", res['Message'])
continue
return gDataStoreClient.commit()
def _storeCommand(self, results):
"""
Stores the results in the cache (SpaceTokenOccupancyCache),
and adds records to the StorageOccupancy accounting.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0}
:returns: S_OK/S_ERROR dict
"""
# Stores in cache
res = self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results["Endpoint"],
lastCheckTime=datetime.utcnow(),
free=results["Free"],
total=results["Total"],
token=results["ElementName"],
)
if not res["OK"]:
self.log.error("Error calling addOrModifySpaceTokenOccupancyCache",
res["Message"])
return res
# Now proceed with the accounting
siteRes = DMSHelpers().getLocalSiteForSE(results["ElementName"])
if not siteRes["OK"]:
return siteRes
accountingDict = {
"StorageElement": results["ElementName"],
"Endpoint": results["Endpoint"],
"Site": siteRes["Value"] if siteRes["Value"] else "unassigned",
}
results["Used"] = results["Total"] - results["Free"]
for sType in ["Total", "Free", "Used"]:
spaceTokenAccounting = StorageOccupancy()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey("SpaceType", sType)
spaceTokenAccounting.setValueByKey(
"Space", int(convertSizeUnits(results[sType], "MB", "B")))
res = gDataStoreClient.addRegister(spaceTokenAccounting)
if not res["OK"]:
self.log.warn("Could not commit register", res["Message"])
continue
return gDataStoreClient.commit()
def _storeCommand(self, results):
""" _storeCommand
Adding records to accounting, on top of what does the derived method.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0,
'SpaceReservation': 'LHCb-Disk'}
:returns: S_OK/S_ERROR dict
"""
res = super(FreeDiskSpaceCommand, self)._storeCommand(results)
if not res['OK']:
return res
siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName'])
if not siteRes['OK']:
return siteRes
if not siteRes['Value']:
return S_OK()
spaceReservation = results.get('SpaceReservation')
accountingDict = {
'SpaceToken': spaceReservation,
'Endpoint': results['Endpoint'],
'Site': siteRes['Value']
}
results['Used'] = results['Total'] - results['Free']
for sType in ['Total', 'Free', 'Used']:
spaceTokenAccounting = SpaceToken()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey('SpaceType', sType)
spaceTokenAccounting.setValueByKey(
'Space', int(convertSizeUnits(results[sType], 'MB', 'B')))
gDataStoreClient.addRegister(spaceTokenAccounting)
gDataStoreClient.commit()
return S_OK()
def initializeStorageElementHandler(serviceInfo):
""" Initialize Storage Element global settings
"""
global BASE_PATH
global USE_TOKENS
global MAX_STORAGE_SIZE
BASE_PATH = getServiceOption(serviceInfo, "BasePath", BASE_PATH)
if not BASE_PATH:
gLogger.error('Failed to get the base path')
return S_ERROR('Failed to get the base path')
mkDir(BASE_PATH)
USE_TOKENS = getServiceOption(serviceInfo, "%UseTokens", USE_TOKENS)
MAX_STORAGE_SIZE = convertSizeUnits(getServiceOption(serviceInfo, "MaxStorageSize", MAX_STORAGE_SIZE), 'MB', 'B')
gLogger.info('Starting DIRAC Storage Element')
gLogger.info('Base Path: %s' % BASE_PATH)
gLogger.info('Max size: %d Bytes' % MAX_STORAGE_SIZE)
gLogger.info('Use access control tokens: ' + str(USE_TOKENS))
return S_OK()
def initializeStorageElementHandler(serviceInfo):
"""Initialize Storage Element global settings"""
global BASE_PATH
global USE_TOKENS
global MAX_STORAGE_SIZE
BASE_PATH = getServiceOption(serviceInfo, "BasePath", "")
if not BASE_PATH:
gLogger.error("Failed to get the base path")
return S_ERROR("Failed to get the base path")
mkDir(BASE_PATH)
USE_TOKENS = getServiceOption(serviceInfo, "UseTokens", USE_TOKENS)
MAX_STORAGE_SIZE = convertSizeUnits(
getServiceOption(serviceInfo, "MaxStorageSize", MAX_STORAGE_SIZE),
"MB", "B")
gLogger.info("Starting DIRAC Storage Element")
gLogger.info("Base Path: %s" % BASE_PATH)
gLogger.info("Max size: %d Bytes" % MAX_STORAGE_SIZE)
gLogger.info("Use access control tokens: " + str(USE_TOKENS))
return S_OK()
def test_convert_error_to_maxint():
""" Make sure that on error we receive -sys.maxint """
assert convertSizeUnits('size', 'B', 'kB') == -sys.maxsize
assert convertSizeUnits(0, 'srcUnit', 'kB') == -sys.maxsize
assert convertSizeUnits(0, 'B', 'dstUnit') == -sys.maxsize
def test_convert_error_to_maxint():
"""Make sure that on error we receive -sys.maxint"""
assert convertSizeUnits("size", "B", "kB") == -sys.maxsize
assert convertSizeUnits(0, "srcUnit", "kB") == -sys.maxsize
assert convertSizeUnits(0, "B", "dstUnit") == -sys.maxsize
def test_convert_error_to_maxint():
""" Make sure that on error we receive -sys.maxint """
assert convertSizeUnits('size', 'B', 'kB') == -sys.maxsize
assert convertSizeUnits(0, 'srcUnit', 'kB') == -sys.maxsize
assert convertSizeUnits(0, 'B', 'dstUnit') == -sys.maxsize
