def initialize( self ):
""" ClientsCacheFeeder initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
VOExtension = getExt()
configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'] )
commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )
commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeeder initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def evaluate(self):
"""
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
"""
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
raise RSSException, "missing 'infoName' in result"
return result
def evaluate(self):
"""
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
"""
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
raise RSSException, "missing 'infoName' in result"
return result
def evaluate(self):
'''
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
'''
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
gLogger.error('missing "infoName" in result')
return None
return result
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def evaluate( self ):
'''
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
'''
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
cc = CommandCaller()
self.command = cc.setCommandObject( self.commandName )
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand( self.command )
self.command.setArgs( self.args )
result = clientsInvoker.doCommand()
if not self.infoName:
result = result[ 'Result' ]
else:
if self.infoName in result.keys():
result = result[ self.infoName ]
else:
gLogger.error( 'missing "infoName" in result' )
return None
return result
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class PolicyCaller:
#############################################################################
def __init__(self, commandCallerIn=None):
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
self.policyInvoker = PolicyInvoker()
#############################################################################
def policyInvocation(
self,
VOExtension,
granularity=None,
name=None,
status=None,
policy=None,
args=None,
pName=None,
pModule=None,
extraArgs=None,
commandIn=None,
):
"""
Invokes a policy:
1. If :attr:`policy` is not None, import the right policy module,
specified with :attr:`VOExtension` (e.g.: 'LHCb') and
:attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
"""
p = policy
a = args
moduleBase = VOExtension + "DIRAC.ResourceStatusSystem.Policy."
if p is None:
try:
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ["*"])
except ImportError:
pModule = "AlwaysFalse_Policy"
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ["*"])
p = getattr(policyModule, pModule)()
if a is None:
a = (granularity, name)
if extraArgs is not None:
if isinstance(extraArgs, tuple):
a = a + extraArgs
elif isinstance(extraArgs, list):
argsList = []
for argsTuple in extraArgs:
argsList.append(a + argsTuple)
a = argsList
if commandIn is not None:
commandIn = self.cc.setCommandObject(commandIn)
res = self._innerEval(p, a, commandIn=commandIn)
res["PolicyName"] = pName
return res
#############################################################################
def _innerEval(self, p, a, commandIn=None, knownInfo=None):
""" policy evaluation
"""
self.policyInvoker.setPolicy(p)
p.setArgs(a)
p.setCommand(commandIn)
# p.setInfoName('Result')
res = self.policyInvoker.evaluatePolicy()
return res
def initialize(self):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsListAccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
commandsVOBOXAvailability = (
'VOBOXAvailabilityCommand',
'VOBOXAvailabilityCommand',
)
commandsSpaceTokenOccupancy = (
'SpaceTokenOccupancyCommand',
'SpaceTokenOccupancyCommand',
)
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [
'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient'
]
knownAPIs = initAPIs(knownAPIs, {})
for command in commandsListClientsCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
self.commandObjectsListClientsCache.append((command, cObj))
for command in commandsListAccountingCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
cArgs = command[2]
self.commandObjectsListAccountingCache.append(
(command, cObj, cArgs))
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject(commandsVOBOXAvailability)
self.commandObjectsVOBOXAvailability.append(
(commandsVOBOXAvailability, cObj, cArgs))
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject(commandsSpaceTokenOccupancy)
self.commandObjectsSpaceTokenOccupancy.append(
(commandsSpaceTokenOccupancy, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
def initialize( self ):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsListAccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', )
commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', )
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
knownAPIs = initAPIs( knownAPIs, {} )
for command in commandsListClientsCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
self.commandObjectsListClientsCache.append( ( command, cObj ) )
for command in commandsListAccountingCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
cArgs = command[ 2 ]
self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) )
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject( commandsVOBOXAvailability )
self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) )
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject( commandsSpaceTokenOccupancy )
self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )
def initialize(self):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsList_AccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
RPCAccounting = RPCClient("Accounting/ReportGenerator")
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command,
cObj,
RPCWMSAdmin=RPCWMSAdmin,
RPCAccounting=RPCAccounting)
self.commandObjectsList_ClientsCache.append((command, cObj))
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append(
(command, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize( self ):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsList_AccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
class PolicyCaller:
#############################################################################
def __init__(self, commandCallerIn=None):
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
self.policyInvoker = PolicyInvoker()
#############################################################################
def policyInvocation(self,
VOExtension,
granularity=None,
name=None,
status=None,
policy=None,
args=None,
pName=None,
pModule=None,
extraArgs=None,
commandIn=None):
"""
Invokes a policy:
1. If :attr:`policy` is not None, import the right policy module,
specified with :attr:`VOExtension` (e.g.: 'LHCb') and
:attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
"""
p = policy
a = args
moduleBase = VOExtension + "DIRAC.ResourceStatusSystem.Policy."
if p is None:
try:
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ['*'])
except ImportError:
pModule = "AlwaysFalse_Policy"
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ['*'])
p = getattr(policyModule, pModule)()
if a is None:
a = (granularity, name)
if extraArgs is not None:
if isinstance(extraArgs, tuple):
a = a + extraArgs
elif isinstance(extraArgs, list):
argsList = []
for argsTuple in extraArgs:
argsList.append(a + argsTuple)
a = argsList
if commandIn is not None:
commandIn = self.cc.setCommandObject(commandIn)
res = self._innerEval(p, a, commandIn=commandIn)
res['PolicyName'] = pName
return res
#############################################################################
def _innerEval(self, p, a, commandIn=None, knownInfo=None):
""" policy evaluation
"""
self.policyInvoker.setPolicy(p)
p.setArgs(a)
p.setCommand(commandIn)
# p.setInfoName('Result')
res = self.policyInvoker.evaluatePolicy()
return res
