class FreeDiskSpaceCommand(Command):
"""
Uses diskSpace method to get the free space
"""
def __init__(self, args=None, clients=None):
super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)
self.rpc = None
self.rsClient = ResourceManagementClient()
def _prepareCommand(self):
"""
FreeDiskSpaceCommand requires one argument:
- name : <str>
"""
if "name" not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args["name"]
return S_OK(elementName)
def doNew(self, masterParams=None):
"""
Gets the total and the free disk space of a DIPS storage element that
is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
"""
if masterParams is not None:
elementName = masterParams
else:
elementName = self._prepareCommand()
if not elementName["OK"]:
return elementName
se = StorageElement(elementName)
elementURL = se.getStorageParameters(protocol="dips")
if elementURL["OK"]:
elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"]
else:
gLogger.verbose("Not a DIPS storage element, skipping...")
return S_OK()
self.rpc = RPCClient(elementURL, timeout=120)
free = self.rpc.getFreeDiskSpace("/")
if not free["OK"]:
return free
free = free["Value"]
total = self.rpc.getTotalDiskSpace("/")
if not total["OK"]:
return total
total = total["Value"]
if free and free < 1:
free = 1
if total and total < 1:
total = 1
result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName
)
if not result["OK"]:
return result
return S_OK()
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancy cache.
"""
elementName = self._prepareCommand()
if not elementName["OK"]:
return elementName
result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName)
if not result["OK"]:
return result
return S_OK(result)
def doMaster(self):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
elements = CSHelpers.getStorageElements()
for name in elements["Value"]:
diskSpace = self.doNew(name)
if not diskSpace["OK"]:
gLogger.error("Unable to calculate free disk space")
continue
return S_OK()
class SpaceTokenOccupancyCommand(Command):
'''
Uses lcg_util to query status of endpoint for a given token.
'''
def __init__(self, args=None, clients=None):
super(SpaceTokenOccupancyCommand, self).__init__(args, clients)
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def _storeCommand(self, results):
'''
Stores the results of doNew method on the database.
'''
for result in results:
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(
result['Endpoint'], result['Token'], result['Total'],
result['Guaranteed'], result['Free'])
if not resQuery['OK']:
return resQuery
return S_OK()
def _prepareCommand(self):
'''
SpaceTokenOccupancy requires one argument:
- elementName : <str>
Given a (storage)elementName, we calculate its endpoint and spaceToken,
which are used to query the srm interface.
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
endpoint = CSHelpers.getStorageElementEndpoint(elementName)
if not endpoint['OK']:
return endpoint
endpoint = endpoint['Value']
spaceToken = CSHelpers.getSEToken(elementName)
if not spaceToken['OK']:
return spaceToken
spaceToken = spaceToken['Value']
return S_OK((endpoint, spaceToken))
def doNew(self, masterParams=None):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
It queries the srm interface, and hopefully it will not crash. Out of the
results, we keep totalsize, guaranteedsuze, and unusedsize.
Then, they are recorded and returned.
'''
if masterParams is not None:
spaceTokenEndpoint, spaceToken = masterParams
else:
params = self._prepareCommand()
if not params['OK']:
return params
spaceTokenEndpoint, spaceToken = params['Value']
# 10 secs of timeout. If it works, the reply is immediate.
occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken,
spaceTokenEndpoint, True, 0)
if not occupancyResult['OK']:
self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \
( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) )
return occupancyResult
else:
occupancy = occupancyResult['Value']
# Timeout does not work here...
# occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )
if occupancy[0] != 0:
return S_ERROR(occupancy)
output = occupancy[1][0]
sTokenDict = {}
sTokenDict['Endpoint'] = spaceTokenEndpoint
sTokenDict['Token'] = spaceToken
sTokenDict['Total'] = float(output.get(
'totalsize', '0')) / 1e12 # Bytes to Terabytes
sTokenDict['Guaranteed'] = float(output.get('guaranteedsize',
'0')) / 1e12
sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12
storeRes = self._storeCommand([sTokenDict])
if not storeRes['OK']:
return storeRes
return S_OK([sTokenDict])
def doCache(self):
'''
Method that reads the cache table and tries to read from it. It will
return a list of dictionaries if there are results.
'''
params = self._prepareCommand()
if not params['OK']:
return params
spaceTokenEndpoint, spaceToken = params['Value']
result = self.rmClient.selectSpaceTokenOccupancyCache(
spaceTokenEndpoint, spaceToken)
if result['OK']:
result = S_OK(
[dict(zip(result['Columns'], res)) for res in result['Value']])
return result
def doMaster(self):
'''
Master method. Gets all endpoints from the storage elements and all
the spaceTokens. Could have taken from Shares/Disk as well.
It queries for all their possible combinations, unless there are records
in the database for those combinations, which then are not queried.
'''
self.log.verbose("Getting all SEs defined in the CS")
storageElementNames = CSHelpers.getStorageElements()
if not storageElementNames['OK']:
self.log.warn(storageElementNames['Message'])
return storageElementNames
storageElementNames = storageElementNames['Value']
endpointTokenSet = set()
for storageElementName in storageElementNames:
endpoint = CSHelpers.getStorageElementEndpoint(storageElementName)
if not endpoint['OK']:
self.log.warn(endpoint['Message'])
continue
endpoint = endpoint['Value']
spaceToken = CSHelpers.getSEToken(storageElementName)
if not spaceToken['OK']:
self.log.warn(spaceToken['Message'])
continue
spaceToken = spaceToken['Value']
endpointTokenSet.add((endpoint, spaceToken))
self.log.verbose('Processing %s' % endpointTokenSet)
for elementToQuery in endpointTokenSet:
result = self.doNew(elementToQuery)
if not result['OK']:
self.metrics['failed'].append(result)
return S_OK(self.metrics)
class FreeDiskSpaceCommand(Command):
'''
Uses diskSpace method to get the free space
'''
def __init__(self, args=None, clients=None):
super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)
self.rmClient = ResourceManagementClient()
def _prepareCommand(self):
'''
FreeDiskSpaceCommand requires one argument:
- name : <str>
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
# We keep TB as default as this is what was used (and will still be used)
# in the policy for "space tokens" ("real", "data" SEs)
unit = self.args.get('unit', 'TB')
return S_OK((elementName, unit))
def doNew(self, masterParams=None):
"""
Gets the parameters to run, either from the master method or from its
own arguments.
Gets the total and the free disk space of a storage element
and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
The result is also returned to the caller, not only inserted.
What is inserted in the DB will normally be in MB,
what is returned will be in the specified unit.
"""
if masterParams is not None:
elementName, unit = masterParams
else:
params = self._prepareCommand()
if not params['OK']:
return params
elementName, unit = params['Value']
endpointResult = CSHelpers.getStorageElementEndpoint(elementName)
if not endpointResult['OK']:
return endpointResult
se = StorageElement(elementName)
occupancyResult = se.getOccupancy(unit=unit)
if not occupancyResult['OK']:
return occupancyResult
occupancy = occupancyResult['Value']
free = occupancy['Free']
total = occupancy['Total']
results = {
'Endpoint': endpointResult['Value'],
'Free': free,
'Total': total,
'ElementName': elementName
}
result = self._storeCommand(results)
if not result['OK']:
return result
return S_OK({'Free': free, 'Total': total})
def _storeCommand(self, results):
""" Here purely for extensibility
"""
return self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results['Endpoint'],
lastCheckTime=datetime.utcnow(),
free=results['Free'],
total=results['Total'],
token=results['ElementName'])
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
It will return a dictionary with th results, converted to "correct" unit.
"""
params = self._prepareCommand()
if not params['OK']:
return params
elementName, unit = params['Value']
result = self.rmClient.selectSpaceTokenOccupancyCache(
token=elementName)
if not result['OK']:
return result
# results are normally in 'MB'
free = result['Value'][0][3]
total = result['Value'][0][4]
free = convertSizeUnits(free, 'MB', unit)
total = convertSizeUnits(total, 'MB', unit)
if free == -sys.maxsize or total == -sys.maxsize:
return S_ERROR("No valid unit specified")
return S_OK({'Free': free, 'Total': total})
def doMaster(self):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
elements = CSHelpers.getStorageElements()
for name in elements['Value']:
# keeping TB as default
diskSpace = self.doNew((name, 'MB'))
if not diskSpace['OK']:
gLogger.warn("Unable to calculate free/total disk space",
"name: %s" % name)
gLogger.warn(diskSpace['Message'])
continue
return S_OK()
class FreeDiskSpaceCommand(Command):
"""
Uses diskSpace method to get the free space
"""
def __init__(self, args=None, clients=None):
super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)
self.rmClient = ResourceManagementClient()
def _prepareCommand(self):
"""
FreeDiskSpaceCommand requires one argument:
- name : <str>
"""
if "name" not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args["name"]
# We keep TB as default as this is what was used (and will still be used)
# in the policy for "space tokens" ("real", "data" SEs)
unit = self.args.get("unit", "TB")
return S_OK((elementName, unit))
def doNew(self, masterParams=None):
"""
Gets the parameters to run, either from the master method or from its
own arguments.
Gets the total and the free disk space of a storage element
and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
The result is also returned to the caller, not only inserted.
What is inserted in the DB will normally be in MB,
what is returned will be in the specified unit.
"""
if masterParams is not None:
elementName, unit = masterParams
else:
params = self._prepareCommand()
if not params["OK"]:
return params
elementName, unit = params["Value"]
se = StorageElement(elementName)
occupancyResult = se.getOccupancy(unit=unit)
if not occupancyResult["OK"]:
return occupancyResult
occupancy = occupancyResult["Value"]
free = occupancy["Free"]
total = occupancy["Total"]
endpointResult = CSHelpers.getStorageElementEndpoint(elementName)
if not endpointResult["OK"]:
return endpointResult
# We only take the first endpoint, in case there are severals of them (which is normal).
# Most probably not ideal, because it would be nice to stay consistent, but well...
endpoint = endpointResult["Value"][0]
results = {
"Endpoint": endpoint,
"Free": free,
"Total": total,
"ElementName": elementName
}
result = self._storeCommand(results)
if not result["OK"]:
return result
return S_OK({"Free": free, "Total": total})
def _storeCommand(self, results):
"""
Stores the results in the cache (SpaceTokenOccupancyCache),
and adds records to the StorageOccupancy accounting.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0}
:returns: S_OK/S_ERROR dict
"""
# Stores in cache
res = self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results["Endpoint"],
lastCheckTime=datetime.utcnow(),
free=results["Free"],
total=results["Total"],
token=results["ElementName"],
)
if not res["OK"]:
self.log.error("Error calling addOrModifySpaceTokenOccupancyCache",
res["Message"])
return res
# Now proceed with the accounting
siteRes = DMSHelpers().getLocalSiteForSE(results["ElementName"])
if not siteRes["OK"]:
return siteRes
accountingDict = {
"StorageElement": results["ElementName"],
"Endpoint": results["Endpoint"],
"Site": siteRes["Value"] if siteRes["Value"] else "unassigned",
}
results["Used"] = results["Total"] - results["Free"]
for sType in ["Total", "Free", "Used"]:
spaceTokenAccounting = StorageOccupancy()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey("SpaceType", sType)
spaceTokenAccounting.setValueByKey(
"Space", int(convertSizeUnits(results[sType], "MB", "B")))
res = gDataStoreClient.addRegister(spaceTokenAccounting)
if not res["OK"]:
self.log.warn("Could not commit register", res["Message"])
continue
return gDataStoreClient.commit()
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
It will return a dictionary with th results, converted to "correct" unit.
"""
params = self._prepareCommand()
if not params["OK"]:
return params
elementName, unit = params["Value"]
result = self.rmClient.selectSpaceTokenOccupancyCache(
token=elementName)
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR(errno.ENODATA, "No occupancy recorded")
# results are normally in 'MB'
free = result["Value"][0][3]
total = result["Value"][0][4]
free = convertSizeUnits(free, "MB", unit)
total = convertSizeUnits(total, "MB", unit)
if free == -sys.maxsize or total == -sys.maxsize:
return S_ERROR("No valid unit specified")
return S_OK({"Free": free, "Total": total})
def doMaster(self):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
for name in DMSHelpers().getStorageElements():
try:
# keeping TB as default
diskSpace = self.doNew((name, "MB"))
if not diskSpace["OK"]:
self.log.warn("Unable to calculate free/total disk space",
"name: %s" % name)
self.log.warn(diskSpace["Message"])
continue
except Exception as excp: # pylint: disable=broad-except
self.log.error(
"Failed to get SE FreeDiskSpace information ==> SE skipped",
name)
self.log.exception("Operation finished with exception: ",
lException=excp)
# Clear the cache
return self._cleanCommand()
def _cleanCommand(self, toDelete=None):
"""Clean the spaceTokenOccupancy table from old endpoints
:param tuple toDelete: endpoint to remove (endpoint, storage_element_name),
e.g. ('httpg://srm-lhcb.cern.ch:8443/srm/managerv2', CERN-RAW)
"""
if not toDelete:
toDelete = []
res = self.rmClient.selectSpaceTokenOccupancyCache()
if not res["OK"]:
return res
storedSEsSet = set([(sse[0], sse[1]) for sse in res["Value"]])
currentSEsSet = set()
currentSEs = DMSHelpers().getStorageElements()
for cse in currentSEs:
res = CSHelpers.getStorageElementEndpoint(cse)
if not res["OK"]:
self.log.warn("Could not get endpoint", res["Message"])
continue
endpoint = res["Value"][0]
currentSEsSet.add((endpoint, cse))
toDelete = list(storedSEsSet - currentSEsSet)
else:
toDelete = [toDelete]
for ep in toDelete:
res = self.rmClient.deleteSpaceTokenOccupancyCache(ep[0], ep[1])
if not res["OK"]:
self.log.warn(
"Could not delete entry from SpaceTokenOccupancyCache",
res["Message"])
return S_OK()
class SpaceTokenOccupancyCommand( Command ):
'''
Uses lcg_util to query status of endpoint for a given token.
'''
def __init__( self, args = None, clients = None ):
super( SpaceTokenOccupancyCommand, self ).__init__( args, clients )
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand( self, results ):
'''
Stores the results of doNew method on the database.
'''
for result in results:
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result[ 'Endpoint' ],
result[ 'Token' ],
result[ 'Total' ],
result[ 'Guaranteed' ],
result[ 'Free' ] )
if not resQuery[ 'OK' ]:
return resQuery
return S_OK()
def _prepareCommand( self ):
'''
SpaceTokenOccupancy requires one argument:
- elementName : <str>
Given a (storage)elementName, we calculate its endpoint and spaceToken,
which are used to query the srm interface.
'''
if not 'name' in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
endpoint = CSHelpers.getStorageElementEndpoint( elementName )
if not endpoint[ 'OK' ]:
return endpoint
endpoint = endpoint[ 'Value' ]
spaceToken = CSHelpers.getStorageElementSpaceToken( elementName )
if not spaceToken[ 'OK' ]:
return spaceToken
spaceToken = spaceToken[ 'Value']
return S_OK( ( endpoint, spaceToken ) )
def doNew( self, masterParams = None ):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
It queries the srm interface, and hopefully it will not crash. Out of the
results, we keep totalsize, guaranteedsuze, and unusedsize.
Then, they are recorded and returned.
'''
if masterParams is not None:
spaceTokenEndpoint, spaceToken = masterParams
else:
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
spaceTokenEndpoint, spaceToken = params[ 'Value' ]
# 10 secs of timeout. If it works, the reply is immediate.
occupancy = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 )
if not occupancy[ 'OK' ]:
return occupancy
occupancy = occupancy[ 'Value' ]
#Timeout does not work here...
#occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )
if occupancy[ 0 ] != 0:
return S_ERROR( occupancy )
output = occupancy[ 1 ][ 0 ]
sTokenDict = {}
sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint
sTokenDict[ 'Token' ] = spaceToken
sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes
sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12
sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12
storeRes = self._storeCommand( [ sTokenDict ] )
if not storeRes[ 'OK' ]:
return storeRes
return S_OK( [ sTokenDict ] )
def doCache( self ):
'''
Method that reads the cache table and tries to read from it. It will
return a list of dictionaries if there are results.
'''
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
spaceTokenEndpoint, spaceToken = params[ 'Value' ]
result = self.rmClient.selectSpaceTokenOccupancyCache( spaceTokenEndpoint, spaceToken )
if result[ 'OK' ]:
result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] )
return result
def doMaster( self ):
'''
Master method. Gets all endpoints from the storage elements and all
the spaceTokens. Could have taken from Shares/Disk as well.
It queries for all their possible combinations, unless there are records
in the database for those combinations, which then are not queried.
'''
spaceTokens = CSHelpers.getSpaceTokens()
if not spaceTokens[ 'OK' ]:
return spaceTokens
spaceTokens = spaceTokens[ 'Value' ]
elementsToCheck = []
seEndpoints = CSHelpers.getStorageElementEndpoints()
if not seEndpoints[ 'OK' ]:
return seEndpoints
seEndpoints = seEndpoints[ 'Value' ]
for seEndpoint in seEndpoints:
for spaceToken in spaceTokens:
elementsToCheck.append( ( seEndpoint, spaceToken ) )
# resQuery = self.rmClient.selectSpaceTokenOccupancyCache( meta = { 'columns' : [ 'Endpoint', 'Token' ] } )
# if not resQuery[ 'OK' ]:
# return resQuery
# resQuery = resQuery[ 'Value' ]
#
# elementsToQuery = list( set( elementsToCheck ).difference( set( resQuery ) ) )
gLogger.verbose( 'Processing %s' % elementsToCheck )
for elementToQuery in elementsToCheck:
result = self.doNew( elementToQuery )
if not result[ 'OK' ]:
self.metrics[ 'failed' ].append( result )
return S_OK( self.metrics )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class SpaceTokenOccupancyCommand(Command):
"""
Uses lcg_util to query status of endpoint for a given token.
"""
def __init__(self, args=None, clients=None):
super(SpaceTokenOccupancyCommand, self).__init__(args, clients)
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand(self, results):
"""
Stores the results of doNew method on the database.
"""
for result in results:
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(
result["Endpoint"], result["Token"], result["Total"], result["Guaranteed"], result["Free"]
)
if not resQuery["OK"]:
return resQuery
return S_OK()
def _prepareCommand(self):
"""
SpaceTokenOccupancy requires one argument:
- elementName : <str>
Given a (storage)elementName, we calculate its endpoint and spaceToken,
which are used to query the srm interface.
"""
if "name" not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args["name"]
endpoint = CSHelpers.getStorageElementEndpoint(elementName)
if not endpoint["OK"]:
return endpoint
endpoint = endpoint["Value"]
spaceToken = CSHelpers.getSEToken(elementName)
if not spaceToken["OK"]:
return spaceToken
spaceToken = spaceToken["Value"]
return S_OK((endpoint, spaceToken))
def doNew(self, masterParams=None):
"""
Gets the parameters to run, either from the master method or from its
own arguments.
It queries the srm interface, and hopefully it will not crash. Out of the
results, we keep totalsize, guaranteedsuze, and unusedsize.
Then, they are recorded and returned.
"""
if masterParams is not None:
spaceTokenEndpoint, spaceToken = masterParams
else:
params = self._prepareCommand()
if not params["OK"]:
return params
spaceTokenEndpoint, spaceToken = params["Value"]
# 10 secs of timeout. If it works, the reply is immediate.
occupancy = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0)
if not occupancy["OK"]:
return occupancy
occupancy = occupancy["Value"]
# Timeout does not work here...
# occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )
if occupancy[0] != 0:
return S_ERROR(occupancy)
output = occupancy[1][0]
sTokenDict = {}
sTokenDict["Endpoint"] = spaceTokenEndpoint
sTokenDict["Token"] = spaceToken
sTokenDict["Total"] = float(output.get("totalsize", "0")) / 1e12 # Bytes to Terabytes
sTokenDict["Guaranteed"] = float(output.get("guaranteedsize", "0")) / 1e12
sTokenDict["Free"] = float(output.get("unusedsize", "0")) / 1e12
storeRes = self._storeCommand([sTokenDict])
if not storeRes["OK"]:
return storeRes
return S_OK([sTokenDict])
def doCache(self):
"""
Method that reads the cache table and tries to read from it. It will
return a list of dictionaries if there are results.
"""
params = self._prepareCommand()
if not params["OK"]:
return params
spaceTokenEndpoint, spaceToken = params["Value"]
result = self.rmClient.selectSpaceTokenOccupancyCache(spaceTokenEndpoint, spaceToken)
if result["OK"]:
result = S_OK([dict(zip(result["Columns"], res)) for res in result["Value"]])
return result
def doMaster(self):
"""
Master method. Gets all endpoints from the storage elements and all
the spaceTokens. Could have taken from Shares/Disk as well.
It queries for all their possible combinations, unless there are records
in the database for those combinations, which then are not queried.
"""
self.log.verbose("Getting all SEs defined in the CS")
storageElementNames = CSHelpers.getStorageElements()
if not storageElementNames["OK"]:
self.log.warn(storageElementNames["Message"])
return storageElementNames
storageElementNames = storageElementNames["Value"]
endpointTokenSet = set()
for storageElementName in storageElementNames:
endpoint = CSHelpers.getStorageElementEndpoint(storageElementName)
if not endpoint["OK"]:
self.log.warn(endpoint["Message"])
continue
endpoint = endpoint["Value"]
spaceToken = CSHelpers.getSEToken(storageElementName)
if not spaceToken["OK"]:
self.log.warn(spaceToken["Message"])
continue
spaceToken = spaceToken["Value"]
endpointTokenSet.add((endpoint, spaceToken))
self.log.verbose("Processing %s" % endpointTokenSet)
for elementToQuery in endpointTokenSet:
result = self.doNew(elementToQuery)
if not result["OK"]:
self.metrics["failed"].append(result)
return S_OK(self.metrics)
class CacheFeederAgent(AgentModule):
'''
The CacheFeederAgent feeds the cache tables for the client and the accounting.
It runs periodically a set of commands, and stores it's results on the
tables.
'''
# Too many public methods
# pylint: disable-msg=R0904
def initialize(self):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsListAccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
commandsVOBOXAvailability = (
'VOBOXAvailabilityCommand',
'VOBOXAvailabilityCommand',
)
commandsSpaceTokenOccupancy = (
'SpaceTokenOccupancyCommand',
'SpaceTokenOccupancyCommand',
)
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [
'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient'
]
knownAPIs = initAPIs(knownAPIs, {})
for command in commandsListClientsCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
self.commandObjectsListClientsCache.append((command, cObj))
for command in commandsListAccountingCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
cArgs = command[2]
self.commandObjectsListAccountingCache.append(
(command, cObj, cArgs))
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject(commandsVOBOXAvailability)
self.commandObjectsVOBOXAvailability.append(
(commandsVOBOXAvailability, cObj, cArgs))
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject(commandsSpaceTokenOccupancy)
self.commandObjectsSpaceTokenOccupancy.append(
(commandsSpaceTokenOccupancy, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
################################################################################
def __getVOBOXAvailabilityCandidates(self):
'''
Gets the candidates to execute the command
'''
# This is horrible, future me, change this.
request_management_urls = gConfig.getValue(
'/Systems/RequestManagement/Development/URLs/allURLS', [])
configuration_urls = gConfig.getServersList()
framework_urls = gConfig.getValue(
'/DIRAC/Framework/SystemAdministrator', [])
elementsToCheck = request_management_urls + configuration_urls + framework_urls
# This may look stupid, but the Command is expecting a tuple
return [(el, ) for el in elementsToCheck]
def __getSpaceTokenOccupancyCandidates(self):
'''
Gets the candidates to execute the command
'''
elementsToCheck = []
spaceEndpoints = CS.getSpaceTokenEndpoints()
spaceTokens = CS.getSpaceTokens()
for site, siteDict in spaceEndpoints.items():
if not isinstance(siteDict, dict):
continue
if not siteDict.has_key('Endpoint'):
continue
for spaceToken in spaceTokens:
elementsToCheck.append((
siteDict['Endpoint'],
spaceToken,
))
return elementsToCheck
def execute(self):
try:
now = datetime.utcnow()
#VOBOX
for co in self.commandObjectsVOBOXAvailability:
commandName = co[0][1].split('_')[0]
self.log.info('Executed %s with %s' %
(commandName, str(co[2])))
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(str(res['Message']))
continue
res = res['Value']
serviceUp = res['serviceUpTime']
machineUp = res['machineUpTime']
site = res['site']
system = res['system']
resQuery = self.rmClient.addOrModifyVOBOXCache(
site, system, serviceUp, machineUp, now)
if not resQuery['OK']:
self.log.error(str(resQuery['Message']))
#SpaceTokenOccupancy
for co in self.commandObjectsSpaceTokenOccupancy:
commandName = co[0][1].split('_')[0]
self.log.info('Executed %s with %s' %
(commandName, str(co[2])))
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(res['Message'])
continue
site, token = co[2]
res = res['Value']
total = res['total']
guaranteed = res['guaranteed']
free = res['free']
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(
site, token, total, guaranteed, free, now)
if not resQuery['OK']:
self.log.error(str(resQuery['Message']))
for co in self.commandObjectsListClientsCache:
commandName = co[0][1].split('_')[0]
self.log.info('Executed %s' % commandName)
try:
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(res['Message'])
continue
res = res['Value']
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug(res)
for key in res.keys():
clientCache = ()
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
clientCache = (key.split()[1], commandName,
res[key]['ID'], value,
res[key][value], None, None)
resQuery = self.rmClient.addOrModifyClientCache(
*clientCache)
if not resQuery['OK']:
self.log.error(resQuery['Message'])
else:
for value in res[key].keys():
clientCache = (key, commandName, None, value,
res[key][value], None, None)
resQuery = self.rmClient.addOrModifyClientCache(
*clientCache)
if not resQuery['OK']:
self.log.error(resQuery['Message'])
except:
self.log.exception("Exception when executing " + co[0][1])
continue
now = datetime.utcnow().replace(microsecond=0)
for co in self.commandObjectsListAccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
commandName = co[0][1].split('_')[0]
plotName = commandName + '_' + str(co[2][0])
self.log.info('Executed %s with args %s %s' %
(commandName, co[0][2], co[0][3]))
try:
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(res['Message'])
continue
res = res['Value']
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug(res)
plotType = res.keys()[0]
if not res[plotType]:
self.log.info(' returned empty...')
self.log.debug(res)
for name in res[plotType].keys():
#name, plotType, plotName, result, dateEffective, lastCheckTime
accountingClient = (name, plotType, plotName,
str(res[plotType][name]), None,
None)
resQuery = self.rmClient.addOrModifyAccountingCache(
*accountingClient)
if not resQuery['OK']:
self.log.error(resQuery['Message'])
except:
self.log.exception("Exception when executing " +
commandName)
continue
return S_OK()
except Exception:
errorStr = "CacheFeederAgent execution"
self.log.exception(errorStr)
return S_ERROR(errorStr)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class CacheFeederAgent( AgentModule ):
'''
The CacheFeederAgent feeds the cache tables for the client and the accounting.
It runs periodically a set of commands, and stores it's results on the
tables.
'''
# Too many public methods
# pylint: disable-msg=R0904
def initialize( self ):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsListAccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', )
commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', )
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
knownAPIs = initAPIs( knownAPIs, {} )
for command in commandsListClientsCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
self.commandObjectsListClientsCache.append( ( command, cObj ) )
for command in commandsListAccountingCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
cArgs = command[ 2 ]
self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) )
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject( commandsVOBOXAvailability )
self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) )
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject( commandsSpaceTokenOccupancy )
self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )
################################################################################
def __getVOBOXAvailabilityCandidates( self ):
'''
Gets the candidates to execute the command
'''
# This is horrible, future me, change this.
request_management_urls = gConfig.getValue( '/Systems/RequestManagement/Development/URLs/allURLS', [] )
configuration_urls = gConfig.getServersList()
framework_urls = gConfig.getValue( '/DIRAC/Framework/SystemAdministrator', [] )
elementsToCheck = request_management_urls + configuration_urls + framework_urls
# This may look stupid, but the Command is expecting a tuple
return [ ( el, ) for el in elementsToCheck ]
def __getSpaceTokenOccupancyCandidates( self ):
'''
Gets the candidates to execute the command
'''
elementsToCheck = []
spaceEndpoints = CS.getSpaceTokenEndpoints()
spaceTokens = CS.getSpaceTokens()
for site,siteDict in spaceEndpoints.items():
if not isinstance( siteDict, dict ):
continue
if not siteDict.has_key( 'Endpoint' ):
continue
for spaceToken in spaceTokens:
elementsToCheck.append( ( siteDict[ 'Endpoint' ], spaceToken, ) )
return elementsToCheck
def execute( self ):
try:
now = datetime.utcnow()
#VOBOX
for co in self.commandObjectsVOBOXAvailability:
commandName = co[0][1].split( '_' )[0]
self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) )
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()[ 'Result' ]
if not res[ 'OK' ]:
self.log.warn( str( res[ 'Message' ] ) )
continue
res = res[ 'Value' ]
serviceUp = res[ 'serviceUpTime' ]
machineUp = res[ 'machineUpTime' ]
site = res[ 'site' ]
system = res[ 'system' ]
resQuery = self.rmClient.addOrModifyVOBOXCache( site, system, serviceUp,
machineUp, now )
if not resQuery[ 'OK' ]:
self.log.error( str( resQuery[ 'Message' ] ) )
#SpaceTokenOccupancy
for co in self.commandObjectsSpaceTokenOccupancy:
commandName = co[0][1].split( '_' )[0]
self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) )
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()[ 'Result' ]
if not res[ 'OK' ]:
self.log.warn( res[ 'Message' ] )
continue
site, token = co[ 2 ]
res = res[ 'Value' ]
total = res[ 'total' ]
guaranteed = res[ 'guaranteed' ]
free = res[ 'free' ]
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( site, token,
total, guaranteed,
free, now )
if not resQuery[ 'OK' ]:
self.log.error( str( resQuery[ 'Message' ] ) )
for co in self.commandObjectsListClientsCache:
commandName = co[0][1].split( '_' )[0]
self.log.info( 'Executed %s' % commandName )
try:
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn( res['Message'] )
continue
res = res[ 'Value' ]
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug( res )
for key in res.keys():
clientCache = ()
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
clientCache = ( key.split()[1], commandName, res[key]['ID'],
value, res[key][value], None, None )
resQuery = self.rmClient.addOrModifyClientCache( *clientCache )
if not resQuery[ 'OK' ]:
self.log.error( resQuery[ 'Message' ] )
else:
for value in res[key].keys():
clientCache = ( key, commandName, None, value,
res[key][value], None, None )
resQuery = self.rmClient.addOrModifyClientCache( *clientCache )
if not resQuery[ 'OK' ]:
self.log.error( resQuery[ 'Message' ] )
except:
self.log.exception( "Exception when executing " + co[0][1] )
continue
now = datetime.utcnow().replace( microsecond = 0 )
for co in self.commandObjectsListAccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
commandName = co[0][1].split( '_' )[0]
plotName = commandName + '_' + str( co[2][0] )
self.log.info( 'Executed %s with args %s %s' % ( commandName, co[0][2], co[0][3] ) )
try:
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn( res['Message'] )
continue
res = res[ 'Value' ]
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug( res )
plotType = res.keys()[ 0 ]
if not res[ plotType ]:
self.log.info(' returned empty...')
self.log.debug( res )
for name in res[ plotType ].keys():
#name, plotType, plotName, result, dateEffective, lastCheckTime
accountingClient = ( name, plotType, plotName, str(res[plotType][name]), None, None )
resQuery = self.rmClient.addOrModifyAccountingCache( *accountingClient )
if not resQuery[ 'OK' ]:
self.log.error( resQuery[ 'Message' ] )
except:
self.log.exception( "Exception when executing " + commandName )
continue
return S_OK()
except Exception:
errorStr = "CacheFeederAgent execution"
self.log.exception( errorStr )
return S_ERROR( errorStr )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class FreeDiskSpaceCommand(Command):
'''
Uses diskSpace method to get the free space
'''
def __init__(self, args=None, clients=None):
super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)
self.rmClient = ResourceManagementClient()
def _prepareCommand(self):
'''
FreeDiskSpaceCommand requires one argument:
- name : <str>
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
# We keep TB as default as this is what was used (and will still be used)
# in the policy for "space tokens" ("real", "data" SEs)
unit = self.args.get('unit', 'TB')
return S_OK((elementName, unit))
def doNew(self, masterParams=None):
"""
Gets the parameters to run, either from the master method or from its
own arguments.
Gets the total and the free disk space of a storage element
and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
The result is also returned to the caller, not only inserted.
What is inserted in the DB will normally be in MB,
what is returned will be in the specified unit.
"""
if masterParams is not None:
elementName, unit = masterParams
else:
params = self._prepareCommand()
if not params['OK']:
return params
elementName, unit = params['Value']
endpointResult = CSHelpers.getStorageElementEndpoint(elementName)
if not endpointResult['OK']:
return endpointResult
se = StorageElement(elementName)
occupancyResult = se.getOccupancy(unit=unit)
if not occupancyResult['OK']:
return occupancyResult
occupancy = occupancyResult['Value']
free = occupancy['Free']
total = occupancy['Total']
results = {'Endpoint': endpointResult['Value'],
'Free': free,
'Total': total,
'ElementName': elementName}
result = self._storeCommand(results)
if not result['OK']:
return result
return S_OK({'Free': free, 'Total': total})
def _storeCommand(self, results):
""" Here purely for extensibility
"""
return self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=results['Endpoint'],
lastCheckTime=datetime.utcnow(),
free=results['Free'],
total=results['Total'],
token=results['ElementName'])
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
It will return a dictionary with th results, converted to "correct" unit.
"""
params = self._prepareCommand()
if not params['OK']:
return params
elementName, unit = params['Value']
result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName)
if not result['OK']:
return result
if not result['Value']:
return S_ERROR(errno.ENODATA, "No occupancy recorded")
# results are normally in 'MB'
free = result['Value'][0][3]
total = result['Value'][0][4]
free = convertSizeUnits(free, 'MB', unit)
total = convertSizeUnits(total, 'MB', unit)
if free == -sys.maxsize or total == -sys.maxsize:
return S_ERROR("No valid unit specified")
return S_OK({'Free': free, 'Total': total})
def doMaster(self):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
elements = CSHelpers.getStorageElements()
for name in elements['Value']:
# keeping TB as default
diskSpace = self.doNew((name, 'MB'))
if not diskSpace['OK']:
gLogger.warn("Unable to calculate free/total disk space", "name: %s" % name)
gLogger.warn(diskSpace['Message'])
continue
return S_OK()
class FreeDiskSpaceCommand( Command ):
'''
Uses diskSpace method to get the free space
'''
def __init__( self, args = None, clients = None ):
super( FreeDiskSpaceCommand, self ).__init__( args, clients = clients )
self.rpc = None
self.rsClient = ResourceManagementClient()
def _prepareCommand( self ):
'''
FreeDiskSpaceCommand requires one argument:
- name : <str>
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
return S_OK( elementName )
def doNew( self, masterParams = None ):
"""
Gets the total and the free disk space of a DIPS storage element that
is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
"""
if masterParams is not None:
elementName = masterParams
else:
elementName = self._prepareCommand()
if not elementName[ 'OK' ]:
return elementName
se = StorageElement(elementName)
elementURL = se.getStorageParameters(protocol = "dips")
if elementURL['OK']:
elementURL = se.getStorageParameters(protocol = "dips")['Value']['URLBase']
else:
gLogger.verbose( "Not a DIPS storage element, skipping..." )
return S_OK()
self.rpc = RPCClient( elementURL, timeout=120 )
free = self.rpc.getFreeDiskSpace("/")
if not free[ 'OK' ]:
return free
free = free['Value']
total = self.rpc.getTotalDiskSpace("/")
if not total[ 'OK' ]:
return total
total = total['Value']
if free and free < 1:
free = 1
if total and total < 1:
total = 1
result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint = elementURL, lastCheckTime = datetime.utcnow(),
free = free, total = total,
token = elementName )
if not result[ 'OK' ]:
return result
return S_OK()
def doCache( self ):
"""
This is a method that gets the element's details from the spaceTokenOccupancy cache.
"""
elementName = self._prepareCommand()
if not elementName[ 'OK' ]:
return elementName
result = self.rsClient.selectSpaceTokenOccupancyCache(token = elementName)
if not result[ 'OK' ]:
return result
return S_OK( result )
def doMaster( self ):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
elements = CSHelpers.getStorageElements()
for name in elements['Value']:
diskSpace = self.doNew( name )
if not diskSpace[ 'OK' ]:
gLogger.error( "Unable to calculate free disk space" )
continue
return S_OK()
