def evaluate(self):
"""
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
"""
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
raise RSSException, "missing 'infoName' in result"
return result
def evaluate(self):
"""
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
"""
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
raise RSSException, "missing 'infoName' in result"
return result
def _innerCall(self, c, a):#, clientIn = None):
""" command call
"""
clientsInvoker = ClientsInvoker()
c.setArgs(a)
# c.setClient(clientIn)
clientsInvoker.setCommand(c)
res = clientsInvoker.doCommand()
return res
#############################################################################
def initialize( self ):
""" ClientsCacheFeeder initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
VOExtension = getExt()
configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'] )
commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )
commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeeder initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def _innerCall(self, c, a): #, clientIn = None):
""" command call
"""
clientsInvoker = ClientsInvoker()
c.setArgs(a)
# c.setClient(clientIn)
clientsInvoker.setCommand(c)
res = clientsInvoker.doCommand()
return res
#############################################################################
def evaluate(self):
'''
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
'''
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
gLogger.error('missing "infoName" in result')
return None
return result
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def evaluate( self ):
'''
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
'''
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
cc = CommandCaller()
self.command = cc.setCommandObject( self.commandName )
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand( self.command )
self.command.setArgs( self.args )
result = clientsInvoker.doCommand()
if not self.infoName:
result = result[ 'Result' ]
else:
if self.infoName in result.keys():
result = result[ self.infoName ]
else:
gLogger.error( 'missing "infoName" in result' )
return None
return result
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class ClientsCacheFeeder( AgentModule ):
#############################################################################
def initialize( self ):
""" ClientsCacheFeeder initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
VOExtension = getExt()
configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'] )
commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )
commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeeder initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
#############################################################################
def execute( self ):
""" The main ClientsCacheFeeder execution method
"""
try:
now = datetime.datetime.utcnow()
for co in self.commandObjectsList_ClientsCache:
try:
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()
for key in res.keys():
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
self.rmDB.addOrModifyClientsCacheRes( key.split()[1], co[0][1].split( '_' )[0],
value, res[key][value], res[key]['ID'] )
else:
for value in res[key].keys():
self.rmDB.addOrModifyClientsCacheRes( key, co[0][1].split( '_' )[0],
value, res[key][value] )
except:
gLogger.exception( "Exception when executing " + co[0][1] )
continue
for co in self.commandObjectsList_AccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
try:
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()
plotType = res.keys()[0]
for name in res[plotType].keys():
plotName = co[0][1].split( '_' )[0] + '_' + str( co[2][0] )
self.rmDB.addOrModifyAccountingCacheRes( name, plotType, plotName,
res[plotType][name] )
except:
gLogger.exception( "Exception when executing " + co[0][1] )
continue
return S_OK()
except Exception:
errorStr = "ClientsCacheFeeder execution"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def initialize(self):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsListAccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
commandsVOBOXAvailability = (
'VOBOXAvailabilityCommand',
'VOBOXAvailabilityCommand',
)
commandsSpaceTokenOccupancy = (
'SpaceTokenOccupancyCommand',
'SpaceTokenOccupancyCommand',
)
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [
'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient'
]
knownAPIs = initAPIs(knownAPIs, {})
for command in commandsListClientsCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
self.commandObjectsListClientsCache.append((command, cObj))
for command in commandsListAccountingCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
cArgs = command[2]
self.commandObjectsListAccountingCache.append(
(command, cObj, cArgs))
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject(commandsVOBOXAvailability)
self.commandObjectsVOBOXAvailability.append(
(commandsVOBOXAvailability, cObj, cArgs))
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject(commandsSpaceTokenOccupancy)
self.commandObjectsSpaceTokenOccupancy.append(
(commandsSpaceTokenOccupancy, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
class CacheFeederAgent(AgentModule):
'''
The CacheFeederAgent feeds the cache tables for the client and the accounting.
It runs periodically a set of commands, and stores it's results on the
tables.
'''
# Too many public methods
# pylint: disable-msg=R0904
def initialize(self):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsListAccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
commandsVOBOXAvailability = (
'VOBOXAvailabilityCommand',
'VOBOXAvailabilityCommand',
)
commandsSpaceTokenOccupancy = (
'SpaceTokenOccupancyCommand',
'SpaceTokenOccupancyCommand',
)
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [
'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient'
]
knownAPIs = initAPIs(knownAPIs, {})
for command in commandsListClientsCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
self.commandObjectsListClientsCache.append((command, cObj))
for command in commandsListAccountingCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
cArgs = command[2]
self.commandObjectsListAccountingCache.append(
(command, cObj, cArgs))
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject(commandsVOBOXAvailability)
self.commandObjectsVOBOXAvailability.append(
(commandsVOBOXAvailability, cObj, cArgs))
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject(commandsSpaceTokenOccupancy)
self.commandObjectsSpaceTokenOccupancy.append(
(commandsSpaceTokenOccupancy, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
################################################################################
def __getVOBOXAvailabilityCandidates(self):
'''
Gets the candidates to execute the command
'''
# This is horrible, future me, change this.
request_management_urls = gConfig.getValue(
'/Systems/RequestManagement/Development/URLs/allURLS', [])
configuration_urls = gConfig.getServersList()
framework_urls = gConfig.getValue(
'/DIRAC/Framework/SystemAdministrator', [])
elementsToCheck = request_management_urls + configuration_urls + framework_urls
# This may look stupid, but the Command is expecting a tuple
return [(el, ) for el in elementsToCheck]
def __getSpaceTokenOccupancyCandidates(self):
'''
Gets the candidates to execute the command
'''
elementsToCheck = []
spaceEndpoints = CS.getSpaceTokenEndpoints()
spaceTokens = CS.getSpaceTokens()
for site, siteDict in spaceEndpoints.items():
if not isinstance(siteDict, dict):
continue
if not siteDict.has_key('Endpoint'):
continue
for spaceToken in spaceTokens:
elementsToCheck.append((
siteDict['Endpoint'],
spaceToken,
))
return elementsToCheck
def execute(self):
try:
now = datetime.utcnow()
#VOBOX
for co in self.commandObjectsVOBOXAvailability:
commandName = co[0][1].split('_')[0]
self.log.info('Executed %s with %s' %
(commandName, str(co[2])))
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(str(res['Message']))
continue
res = res['Value']
serviceUp = res['serviceUpTime']
machineUp = res['machineUpTime']
site = res['site']
system = res['system']
resQuery = self.rmClient.addOrModifyVOBOXCache(
site, system, serviceUp, machineUp, now)
if not resQuery['OK']:
self.log.error(str(resQuery['Message']))
#SpaceTokenOccupancy
for co in self.commandObjectsSpaceTokenOccupancy:
commandName = co[0][1].split('_')[0]
self.log.info('Executed %s with %s' %
(commandName, str(co[2])))
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(res['Message'])
continue
site, token = co[2]
res = res['Value']
total = res['total']
guaranteed = res['guaranteed']
free = res['free']
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(
site, token, total, guaranteed, free, now)
if not resQuery['OK']:
self.log.error(str(resQuery['Message']))
for co in self.commandObjectsListClientsCache:
commandName = co[0][1].split('_')[0]
self.log.info('Executed %s' % commandName)
try:
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(res['Message'])
continue
res = res['Value']
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug(res)
for key in res.keys():
clientCache = ()
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
clientCache = (key.split()[1], commandName,
res[key]['ID'], value,
res[key][value], None, None)
resQuery = self.rmClient.addOrModifyClientCache(
*clientCache)
if not resQuery['OK']:
self.log.error(resQuery['Message'])
else:
for value in res[key].keys():
clientCache = (key, commandName, None, value,
res[key][value], None, None)
resQuery = self.rmClient.addOrModifyClientCache(
*clientCache)
if not resQuery['OK']:
self.log.error(resQuery['Message'])
except:
self.log.exception("Exception when executing " + co[0][1])
continue
now = datetime.utcnow().replace(microsecond=0)
for co in self.commandObjectsListAccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
commandName = co[0][1].split('_')[0]
plotName = commandName + '_' + str(co[2][0])
self.log.info('Executed %s with args %s %s' %
(commandName, co[0][2], co[0][3]))
try:
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn(res['Message'])
continue
res = res['Value']
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug(res)
plotType = res.keys()[0]
if not res[plotType]:
self.log.info(' returned empty...')
self.log.debug(res)
for name in res[plotType].keys():
#name, plotType, plotName, result, dateEffective, lastCheckTime
accountingClient = (name, plotType, plotName,
str(res[plotType][name]), None,
None)
resQuery = self.rmClient.addOrModifyAccountingCache(
*accountingClient)
if not resQuery['OK']:
self.log.error(resQuery['Message'])
except:
self.log.exception("Exception when executing " +
commandName)
continue
return S_OK()
except Exception:
errorStr = "CacheFeederAgent execution"
self.log.exception(errorStr)
return S_ERROR(errorStr)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def initialize( self ):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsListAccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', )
commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', )
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
knownAPIs = initAPIs( knownAPIs, {} )
for command in commandsListClientsCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
self.commandObjectsListClientsCache.append( ( command, cObj ) )
for command in commandsListAccountingCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
cArgs = command[ 2 ]
self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) )
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject( commandsVOBOXAvailability )
self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) )
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject( commandsSpaceTokenOccupancy )
self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )
class CacheFeederAgent( AgentModule ):
'''
The CacheFeederAgent feeds the cache tables for the client and the accounting.
It runs periodically a set of commands, and stores it's results on the
tables.
'''
# Too many public methods
# pylint: disable-msg=R0904
def initialize( self ):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsListAccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', )
commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', )
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
knownAPIs = initAPIs( knownAPIs, {} )
for command in commandsListClientsCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
self.commandObjectsListClientsCache.append( ( command, cObj ) )
for command in commandsListAccountingCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
cArgs = command[ 2 ]
self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) )
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject( commandsVOBOXAvailability )
self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) )
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject( commandsSpaceTokenOccupancy )
self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )
################################################################################
def __getVOBOXAvailabilityCandidates( self ):
'''
Gets the candidates to execute the command
'''
# This is horrible, future me, change this.
request_management_urls = gConfig.getValue( '/Systems/RequestManagement/Development/URLs/allURLS', [] )
configuration_urls = gConfig.getServersList()
framework_urls = gConfig.getValue( '/DIRAC/Framework/SystemAdministrator', [] )
elementsToCheck = request_management_urls + configuration_urls + framework_urls
# This may look stupid, but the Command is expecting a tuple
return [ ( el, ) for el in elementsToCheck ]
def __getSpaceTokenOccupancyCandidates( self ):
'''
Gets the candidates to execute the command
'''
elementsToCheck = []
spaceEndpoints = CS.getSpaceTokenEndpoints()
spaceTokens = CS.getSpaceTokens()
for site,siteDict in spaceEndpoints.items():
if not isinstance( siteDict, dict ):
continue
if not siteDict.has_key( 'Endpoint' ):
continue
for spaceToken in spaceTokens:
elementsToCheck.append( ( siteDict[ 'Endpoint' ], spaceToken, ) )
return elementsToCheck
def execute( self ):
try:
now = datetime.utcnow()
#VOBOX
for co in self.commandObjectsVOBOXAvailability:
commandName = co[0][1].split( '_' )[0]
self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) )
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()[ 'Result' ]
if not res[ 'OK' ]:
self.log.warn( str( res[ 'Message' ] ) )
continue
res = res[ 'Value' ]
serviceUp = res[ 'serviceUpTime' ]
machineUp = res[ 'machineUpTime' ]
site = res[ 'site' ]
system = res[ 'system' ]
resQuery = self.rmClient.addOrModifyVOBOXCache( site, system, serviceUp,
machineUp, now )
if not resQuery[ 'OK' ]:
self.log.error( str( resQuery[ 'Message' ] ) )
#SpaceTokenOccupancy
for co in self.commandObjectsSpaceTokenOccupancy:
commandName = co[0][1].split( '_' )[0]
self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) )
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()[ 'Result' ]
if not res[ 'OK' ]:
self.log.warn( res[ 'Message' ] )
continue
site, token = co[ 2 ]
res = res[ 'Value' ]
total = res[ 'total' ]
guaranteed = res[ 'guaranteed' ]
free = res[ 'free' ]
resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( site, token,
total, guaranteed,
free, now )
if not resQuery[ 'OK' ]:
self.log.error( str( resQuery[ 'Message' ] ) )
for co in self.commandObjectsListClientsCache:
commandName = co[0][1].split( '_' )[0]
self.log.info( 'Executed %s' % commandName )
try:
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn( res['Message'] )
continue
res = res[ 'Value' ]
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug( res )
for key in res.keys():
clientCache = ()
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
clientCache = ( key.split()[1], commandName, res[key]['ID'],
value, res[key][value], None, None )
resQuery = self.rmClient.addOrModifyClientCache( *clientCache )
if not resQuery[ 'OK' ]:
self.log.error( resQuery[ 'Message' ] )
else:
for value in res[key].keys():
clientCache = ( key, commandName, None, value,
res[key][value], None, None )
resQuery = self.rmClient.addOrModifyClientCache( *clientCache )
if not resQuery[ 'OK' ]:
self.log.error( resQuery[ 'Message' ] )
except:
self.log.exception( "Exception when executing " + co[0][1] )
continue
now = datetime.utcnow().replace( microsecond = 0 )
for co in self.commandObjectsListAccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
commandName = co[0][1].split( '_' )[0]
plotName = commandName + '_' + str( co[2][0] )
self.log.info( 'Executed %s with args %s %s' % ( commandName, co[0][2], co[0][3] ) )
try:
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()['Result']
if not res['OK']:
self.log.warn( res['Message'] )
continue
res = res[ 'Value' ]
if not res or res is None:
self.log.info(' returned empty...')
continue
self.log.debug( res )
plotType = res.keys()[ 0 ]
if not res[ plotType ]:
self.log.info(' returned empty...')
self.log.debug( res )
for name in res[ plotType ].keys():
#name, plotType, plotName, result, dateEffective, lastCheckTime
accountingClient = ( name, plotType, plotName, str(res[plotType][name]), None, None )
resQuery = self.rmClient.addOrModifyAccountingCache( *accountingClient )
if not resQuery[ 'OK' ]:
self.log.error( resQuery[ 'Message' ] )
except:
self.log.exception( "Exception when executing " + commandName )
continue
return S_OK()
except Exception:
errorStr = "CacheFeederAgent execution"
self.log.exception( errorStr )
return S_ERROR( errorStr )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def initialize(self):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsList_AccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
RPCAccounting = RPCClient("Accounting/ReportGenerator")
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command,
cObj,
RPCWMSAdmin=RPCWMSAdmin,
RPCAccounting=RPCAccounting)
self.commandObjectsList_ClientsCache.append((command, cObj))
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append(
(command, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
class ClientsCacheFeederAgent(AgentModule):
#############################################################################
def initialize(self):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsList_AccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
RPCAccounting = RPCClient("Accounting/ReportGenerator")
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command,
cObj,
RPCWMSAdmin=RPCWMSAdmin,
RPCAccounting=RPCAccounting)
self.commandObjectsList_ClientsCache.append((command, cObj))
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append(
(command, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
#############################################################################
def execute(self):
""" The main ClientsCacheFeederAgent execution method
"""
try:
now = datetime.datetime.utcnow()
for co in self.commandObjectsList_ClientsCache:
try:
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()
for key in res.keys():
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
self.rmDB.addOrModifyClientsCacheRes(
key.split()[1], co[0][1].split('_')[0],
value, res[key][value], res[key]['ID'])
else:
for value in res[key].keys():
self.rmDB.addOrModifyClientsCacheRes(
key, co[0][1].split('_')[0], value,
res[key][value])
except:
gLogger.exception("Exception when executing " + co[0][1])
continue
for co in self.commandObjectsList_AccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
try:
co[1].setArgs(co[2])
self.clientsInvoker.setCommand(co[1])
res = self.clientsInvoker.doCommand()
plotType = res.keys()[0]
for name in res[plotType].keys():
plotName = co[0][1].split('_')[0] + '_' + str(co[2][0])
self.rmDB.addOrModifyAccountingCacheRes(
name, plotType, plotName, res[plotType][name])
except:
gLogger.exception("Exception when executing " + co[0][1])
continue
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent execution"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize( self ):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsList_AccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
class ClientsCacheFeederAgent( AgentModule ):
#############################################################################
def initialize( self ):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsList_AccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
#############################################################################
def execute( self ):
""" The main ClientsCacheFeederAgent execution method
"""
try:
now = datetime.datetime.utcnow()
for co in self.commandObjectsList_ClientsCache:
try:
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()
for key in res.keys():
if 'ID' in res[key].keys():
for value in res[key].keys():
if value != 'ID':
self.rmDB.addOrModifyClientsCacheRes( key.split()[1], co[0][1].split( '_' )[0],
value, res[key][value], res[key]['ID'] )
else:
for value in res[key].keys():
self.rmDB.addOrModifyClientsCacheRes( key, co[0][1].split( '_' )[0],
value, res[key][value] )
except:
gLogger.exception( "Exception when executing " + co[0][1] )
continue
for co in self.commandObjectsList_AccountingCache:
if co[0][3] == 'Hourly':
if now.minute >= 10:
continue
elif co[0][3] == 'Daily':
if now.hour >= 1:
continue
try:
co[1].setArgs( co[2] )
self.clientsInvoker.setCommand( co[1] )
res = self.clientsInvoker.doCommand()
plotType = res.keys()[ 0 ]
for name in res[ plotType ].keys():
plotName = co[0][1].split( '_' )[0] + '_' + str( co[2][0] )
self.rmDB.addOrModifyAccountingCacheRes( name, plotType, plotName,
res[plotType][name] )
except:
gLogger.exception( "Exception when executing " + co[ 0 ][ 1 ] )
continue
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent execution"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
