def evaluate(self):
"""
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
"""
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
raise RSSException, "missing 'infoName' in result"
return result
def evaluate(self):
"""
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
"""
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
raise RSSException, "missing 'infoName' in result"
return result
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def __init__(self, commandCallerIn=None):
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
self.policyInvoker = PolicyInvoker()
def initializeResourceManagementHandler(serviceInfo):
global rsDB
rsDB = ResourceStatusDB()
global rmDB
rmDB = ResourceManagementDB()
cc = CommandCaller()
global VOExtension
VOExtension = getExt()
ig = InfoGetter(VOExtension)
WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
global publisher
publisher = Publisher(VOExtension,
rsDBIn=rsDB,
commandCallerIn=cc,
infoGetterIn=ig,
WMSAdminIn=WMSAdmin)
# sync_O = Synchronizer(rsDB)
# gConfig.addListenerToNewVersionEvent( sync_O.sync )
return S_OK()
def __init__(self, commandCallerIn=None, **clients):
'''
Constructor
'''
if commandCallerIn is None:
commandCallerIn = CommandCaller()
self.cCaller = commandCallerIn
self.clients = clients
def __init__(self, commandCallerIn=None):
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
self.policyInvoker = PolicyInvoker()
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
WMSAdminIn = None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool( 2, 5 )
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def initialize( self ):
""" ClientsCacheFeeder initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
VOExtension = getExt()
configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'] )
commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )
commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeeder initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def evaluate(self):
'''
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
'''
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
cc = CommandCaller()
self.command = cc.setCommandObject(self.commandName)
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand(self.command)
self.command.setArgs(self.args)
result = clientsInvoker.doCommand()
if not self.infoName:
result = result['Result']
else:
if self.infoName in result.keys():
result = result[self.infoName]
else:
gLogger.error('missing "infoName" in result')
return None
return result
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def evaluate( self ):
'''
Before use, call at least `setArgs` and, alternatively,
`setCommand` or `setCommandName`.
Invoking `super(PolicyCLASS, self).evaluate` will invoke
the command (if necessary) as it is provided and returns the results.
'''
if self.knownInfo:
result = self.knownInfo
else:
if not self.command:
# use standard Command
cc = CommandCaller()
self.command = cc.setCommandObject( self.commandName )
clientsInvoker = ClientsInvoker()
clientsInvoker.setCommand( self.command )
self.command.setArgs( self.args )
result = clientsInvoker.doCommand()
if not self.infoName:
result = result[ 'Result' ]
else:
if self.infoName in result.keys():
result = result[ self.infoName ]
else:
gLogger.error( 'missing "infoName" in result' )
return None
return result
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__(self, **clients):
'''
Constructor. Defines members that will be used later on.
'''
cc = CommandCaller()
self.clients = clients
self.pCaller = PolicyCaller(cc, **clients)
self.iGetter = InfoGetter()
self.__granularity = None
self.__name = None
self.__statusType = None
self.__status = None
self.__formerStatus = None
self.__reason = None
self.__siteType = None
self.__serviceType = None
self.__resourceType = None
self.__useNewRes = None
def __init__(self,
VOExtension,
granularity=None,
name=None,
status=None,
formerStatus=None,
reason=None,
siteType=None,
serviceType=None,
resourceType=None,
useNewRes=False):
"""
PDP (Policy Decision Point) initialization
:params:
:attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
:attr:`granularity`: string - a ValidRes
:attr:`name`: string - name (e.g. of a site)
:attr:`status`: string - status
:attr:`formerStatus`: string - former status
:attr:`reason`: string - optional reason for last status change
:attr:`siteType`: string - optional site type
:attr:`serviceType`: string - optional service type
:attr:`resourceType`: string - optional resource type
"""
self.VOExtension = VOExtension
self.__granularity = assignOrRaise(granularity, ValidRes, InvalidRes,
self, self.__init__)
self.__name = name
self.__status = assignOrRaise(status, ValidStatus, InvalidStatus, self,
self.__init__)
self.__formerStatus = assignOrRaise(formerStatus, ValidStatus,
InvalidStatus, self, self.__init__)
self.__reason = reason
self.__siteType = assignOrRaise(siteType, ValidSiteType,
InvalidSiteType, self, self.__init__)
self.__serviceType = assignOrRaise(serviceType, ValidServiceType,
InvalidServiceType, self,
self.__init__)
self.__resourceType = assignOrRaise(resourceType, ValidResourceType,
InvalidResourceType, self,
self.__init__)
cc = CommandCaller()
self.pc = PolicyCaller(cc)
self.useNewRes = useNewRes
self.args = None
self.policy = None
self.knownInfo = None
self.ig = None
class PolicyCaller:
#############################################################################
def __init__(self, commandCallerIn=None):
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
self.policyInvoker = PolicyInvoker()
#############################################################################
def policyInvocation(
self,
VOExtension,
granularity=None,
name=None,
status=None,
policy=None,
args=None,
pName=None,
pModule=None,
extraArgs=None,
commandIn=None,
):
"""
Invokes a policy:
1. If :attr:`policy` is not None, import the right policy module,
specified with :attr:`VOExtension` (e.g.: 'LHCb') and
:attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
"""
p = policy
a = args
moduleBase = VOExtension + "DIRAC.ResourceStatusSystem.Policy."
if p is None:
try:
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ["*"])
except ImportError:
pModule = "AlwaysFalse_Policy"
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ["*"])
p = getattr(policyModule, pModule)()
if a is None:
a = (granularity, name)
if extraArgs is not None:
if isinstance(extraArgs, tuple):
a = a + extraArgs
elif isinstance(extraArgs, list):
argsList = []
for argsTuple in extraArgs:
argsList.append(a + argsTuple)
a = argsList
if commandIn is not None:
commandIn = self.cc.setCommandObject(commandIn)
res = self._innerEval(p, a, commandIn=commandIn)
res["PolicyName"] = pName
return res
#############################################################################
def _innerEval(self, p, a, commandIn=None, knownInfo=None):
""" policy evaluation
"""
self.policyInvoker.setPolicy(p)
p.setArgs(a)
p.setCommand(commandIn)
# p.setInfoName('Result')
res = self.policyInvoker.evaluatePolicy()
return res
class Publisher:
"""
Class Publisher is in charge of getting dispersed information, to be published on the web.
"""
#############################################################################
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
WMSAdminIn = None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool( 2, 5 )
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
#############################################################################
def getInfo(self, granularity, name, useNewRes = False):
"""
Standard method to get all the info to be published
This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
with 2-5 threads. The threaded method is
:meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`
:params:
:attr:`granularity`: string - a ValidRes
:attr:`name`: string - name of the Validres
:attr:`useNewRes`: boolean. When set to true, will get new results,
otherwise it will get cached results (where available).
"""
if granularity not in ValidRes:
raise InvalidRes, Utils.where(self, self.getInfo)
self.infoForPanel_res = {}
status = None
formerStatus = None
siteType = None
serviceType = None
resourceType = None
if granularity in ('Resource', 'Resources'):
try:
resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'],
resourceName = name)[0][0]
except IndexError:
return "%s does not exist!" %name
if granularity in ('StorageElement', 'StorageElements'):
try:
siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'],
storageElementName = name)[0][0]
except IndexError:
return "%s does not exist!" %name
paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status',
'RSS Status', 'Reason', 'Description']
infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status,
formerStatus = formerStatus, siteType = siteType,
serviceType = serviceType, resourceType = resourceType,
useNewRes = useNewRes)[0]['Panels']
infoToGet_res = {}
recordsList = []
infosForPolicy = {}
for panel in infoToGet.keys():
(granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel)
if not self._resExist(granularityForPanel, nameForPanel):
# completeInfoForPanel_res = None
continue
#take composite RSS result for name
nameStatus_res = self._getStatus(nameForPanel, panel)
recordBase = [None, None, None, None, None, None, None, None]
recordBase[1] = panel.replace('_Panel', '')
recordBase[2] = nameForPanel #nameForPanel
try:
recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status
except:
pass
recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status
record = copy.deepcopy(recordBase)
record[0] = 'ResultsForResource'
recordsList.append(record)
#take info that goes into the panel
infoForPanel = infoToGet[panel]
for info in infoForPanel:
self.threadPool.generateJobAndQueueIt(self.getInfoForPanel,
args = (info, granularityForPanel, nameForPanel) )
self.threadPool.processAllResults()
for policy in [x.keys()[0] for x in infoForPanel]:
record = copy.deepcopy(recordBase)
record[0] = 'SpecificInformation'
record[3] = policy #policyName
record[4] = None #DIRAC Status
record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy
record[6] = self.infoForPanel_res[policy]['Reason'] #Reason
record[7] = self.infoForPanel_res[policy]['desc'] #Description
recordsList.append(record)
infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']
infoToGet_res['TotalRecords'] = len(recordsList)
infoToGet_res['ParameterNames'] = paramNames
infoToGet_res['Records'] = recordsList
infoToGet_res['Extras'] = infosForPolicy
return infoToGet_res
#############################################################################
def getInfoForPanel(self, info, granularityForPanel, nameForPanel):
#get single RSS policy results
policyResToGet = info.keys()[0]
pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet)
if pol_res != []:
pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]}
else:
pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'}
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet] = pol_res_dict
finally:
self.lockObj.release()
#get policy description
desc = self._getPolicyDesc(policyResToGet)
#get other info
othersInfo = info.values()[0]
if not isinstance(othersInfo, list):
othersInfo = [othersInfo]
info_res = {}
for oi in othersInfo:
format_ = oi.keys()[0]
what = oi.values()[0]
info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what)
info_res[format_] = info_bit_got
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet]['infos'] = info_res
self.infoForPanel_res[policyResToGet]['desc'] = desc
finally:
self.lockObj.release()
#############################################################################
def _getStatus(self, name, panel):
#get RSS status
RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]
#get DIRAC status
if panel in ('Site_Panel', 'SE_Panel'):
if panel == 'Site_Panel':
DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
if DIRACStatus['OK']:
DIRACStatus = DIRACStatus['Value'][name].pop()[0]
else:
raise RSSException, Utils.where(self, self._getStatus)
elif panel == 'SE_Panel':
ra = getStorageElementStatus(name, 'ReadAccess')['Value']
wa = getStorageElementStatus(name, 'WriteAccess')['Value']
DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}
status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } }
else:
status = { name : { 'RSSStatus': RSSStatus} }
return status
#############################################################################
def _getInfo(self, granularity, name, format_, what):
if format_ == 'RSS':
info_bit_got = self._getInfoFromRSSDB(name, what)
else:
if isinstance(what, dict):
command = what['CommandIn']
extraArgs = what['args']
else:
command = what
extraArgs = None
info_bit_got = self.cc.commandInvocation(granularity, name, None,
None, command, extraArgs)
try:
info_bit_got = info_bit_got['Result']
except:
pass
return info_bit_got
#############################################################################
def _getInfoFromRSSDB(self, name, what):
paramsL = ['Status']
siteName = None
serviceName = None
resourceName = None
storageElementName = None
serviceType = None
gridSiteName = None
if what == 'ServiceOfSite':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
paramsL.append('Reason')
siteName = name
elif what == 'ResOfCompService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorEl':
gran = 'StorageElements'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
storageElementName = name
elif what == 'StorageElementsOfSite':
gran = 'StorageElements'
paramsL.insert(0, 'StorageElementName')
paramsL.append('Reason')
if '@' in name:
DIRACsiteName = name.split('@').pop()
else:
DIRACsiteName = name
gridSiteName = getGOCSiteName(DIRACsiteName)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'Site_Panel':
gran = 'Site'
paramsL.insert(0, 'SiteName')
siteName = name
elif what == 'Service_Computing_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_Storage_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VO-BOX_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VOMS_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Resource_Panel':
gran = 'Resource'
paramsL.insert(0, 'ResourceName')
resourceName = name
elif what == 'SE_Panel':
gran = 'StorageElement'
paramsL.insert(0, 'StorageElementName')
storageElementName = name
info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName,
serviceName = serviceName, serviceType = serviceType,
resourceName = resourceName,
storageElementName = storageElementName,
gridSiteName = gridSiteName)
return info_bit_got
#############################################################################
def _getPolicyDesc(self, policyName):
return self.configModule.Policies[policyName]['Description']
#############################################################################
def __getNameForPanel(self, granularity, name, panel):
if granularity in ('Site', 'Sites'):
if panel == 'Service_Computing_Panel':
granularity = 'Service'
name = 'Computing@' + name
elif panel == 'Service_Storage_Panel':
granularity = 'Service'
name = 'Storage@' + name
elif panel == 'OtherServices_Panel':
granularity = 'Service'
name = 'OtherS@' + name
elif panel == 'Service_VOMS_Panel':
granularity = 'Service'
name = 'VOMS@' + name
elif panel == 'Service_VO-BOX_Panel':
granularity = 'Service'
name = 'VO-BOX@' + name
# else:
# granularity = granularity
# name = name
# else:
# granularity = granularity
# name = name
return (granularity, name)
#############################################################################
def _resExist(self, granularity, name):
siteName = None
serviceName = None
resourceName = None
storageElementName = None
if granularity in ('Site', 'Sites'):
siteName = name
elif granularity in ('Service', 'Services'):
serviceName = name
elif granularity in ('Resource', 'Resources'):
resourceName = name
elif granularity in ('StorageElement', 'StorageElements'):
storageElementName = name
res = self.rsDB.getMonitoredsList(granularity, siteName = siteName,
serviceName = serviceName, resourceName = resourceName,
storageElementName = storageElementName)
if res == []:
return False
else:
return True
class Publisher:
"""
Class Publisher is in charge of getting dispersed information, to be published on the web.
"""
#############################################################################
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = __import__(
VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'])
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
#############################################################################
def getInfo(self, granularity, name, useNewRes=False):
"""
Standard method to get all the info to be published
This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
with 2-5 threads. The threaded method is
:meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`
:params:
:attr:`granularity`: string - a ValidRes
:attr:`name`: string - name of the Validres
:attr:`useNewRes`: boolean. When set to true, will get new results,
otherwise it will get cached results (where available).
"""
if granularity not in ValidRes:
raise InvalidRes, where(self, self.getInfo)
self.infoForPanel_res = {}
status = None
formerStatus = None
siteType = None
serviceType = None
resourceType = None
if granularity in ('Resource', 'Resources'):
try:
resourceType = self.rsDB.getMonitoredsList(
'Resource', ['ResourceType'], resourceName=name)[0][0]
except IndexError:
return "%s does not exist!" % name
if granularity in ('StorageElement', 'StorageElements'):
try:
siteType = self.rsDB.getMonitoredsList(
'StorageElement', ['SiteType'],
storageElementName=name)[0][0]
except IndexError:
return "%s does not exist!" % name
paramNames = [
'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status',
'Reason', 'Description'
]
infoToGet = self.ig.getInfoToApply(('view_info', ),
granularity,
status=status,
formerStatus=formerStatus,
siteType=siteType,
serviceType=serviceType,
resourceType=resourceType,
useNewRes=useNewRes)[0]['Panels']
infoToGet_res = {}
recordsList = []
infosForPolicy = {}
for panel in infoToGet.keys():
(granularityForPanel,
nameForPanel) = self.__getNameForPanel(granularity, name, panel)
if not self._resExist(granularityForPanel, nameForPanel):
# completeInfoForPanel_res = None
continue
#take composite RSS result for name
nameStatus_res = self._getStatus(nameForPanel, panel)
recordBase = [None, None, None, None, None, None, None, None]
recordBase[1] = panel.replace('_Panel', '')
recordBase[2] = nameForPanel #nameForPanel
try:
recordBase[4] = nameStatus_res[nameForPanel][
'DIRACStatus'] #DIRAC Status
except:
pass
recordBase[5] = nameStatus_res[nameForPanel][
'RSSStatus'] #RSS Status
record = copy.deepcopy(recordBase)
record[0] = 'ResultsForResource'
recordsList.append(record)
#take info that goes into the panel
infoForPanel = infoToGet[panel]
for info in infoForPanel:
self.threadPool.generateJobAndQueueIt(
self.getInfoForPanel,
args=(info, granularityForPanel, nameForPanel))
self.threadPool.processAllResults()
for policy in [x.keys()[0] for x in infoForPanel]:
record = copy.deepcopy(recordBase)
record[0] = 'SpecificInformation'
record[3] = policy #policyName
record[4] = None #DIRAC Status
record[5] = self.infoForPanel_res[policy][
'Status'] #RSS status for the policy
record[6] = self.infoForPanel_res[policy]['Reason'] #Reason
record[7] = self.infoForPanel_res[policy]['desc'] #Description
recordsList.append(record)
infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']
infoToGet_res['TotalRecords'] = len(recordsList)
infoToGet_res['ParameterNames'] = paramNames
infoToGet_res['Records'] = recordsList
infoToGet_res['Extras'] = infosForPolicy
return infoToGet_res
#############################################################################
def getInfoForPanel(self, info, granularityForPanel, nameForPanel):
#get single RSS policy results
policyResToGet = info.keys()[0]
pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet)
if pol_res != []:
pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]}
else:
pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'}
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet] = pol_res_dict
finally:
self.lockObj.release()
#get policy description
desc = self._getPolicyDesc(policyResToGet)
#get other info
othersInfo = info.values()[0]
if not isinstance(othersInfo, list):
othersInfo = [othersInfo]
info_res = {}
for oi in othersInfo:
format = oi.keys()[0]
what = oi.values()[0]
info_bit_got = self._getInfo(granularityForPanel, nameForPanel,
format, what)
info_res[format] = info_bit_got
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet]['infos'] = info_res
self.infoForPanel_res[policyResToGet]['desc'] = desc
finally:
self.lockObj.release()
#############################################################################
def _getStatus(self, name, panel):
#get RSS status
RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]
#get DIRAC status
if panel in ('Site_Panel', 'SE_Panel'):
if panel == 'Site_Panel':
DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
if DIRACStatus['OK']:
DIRACStatus = DIRACStatus['Value'][name].pop()[0]
else:
raise RSSException, where(self, self._getStatus)
elif panel == 'SE_Panel':
ra = getStorageElementStatus(name, 'ReadAccess')['Value']
wa = getStorageElementStatus(name, 'WriteAccess')['Value']
DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}
status = {
name: {
'RSSStatus': RSSStatus,
'DIRACStatus': DIRACStatus
}
}
else:
status = {name: {'RSSStatus': RSSStatus}}
return status
#############################################################################
def _getInfo(self, granularity, name, format, what):
if format == 'RSS':
info_bit_got = self._getInfoFromRSSDB(name, what)
else:
if isinstance(what, dict):
command = what['CommandIn']
extraArgs = what['args']
else:
command = what
extraArgs = None
info_bit_got = self.cc.commandInvocation(granularity, name, None,
None, command, extraArgs)
try:
info_bit_got = info_bit_got['Result']
except:
pass
return info_bit_got
#############################################################################
def _getInfoFromRSSDB(self, name, what):
paramsL = ['Status']
siteName = None
serviceName = None
resourceName = None
storageElementName = None
serviceType = None
gridSiteName = None
if what == 'ServiceOfSite':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
paramsL.append('Reason')
siteName = name
elif what == 'ResOfCompService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorEl':
gran = 'StorageElements'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
storageElementName = name
elif what == 'StorageElementsOfSite':
gran = 'StorageElements'
paramsL.insert(0, 'StorageElementName')
paramsL.append('Reason')
if '@' in name:
DIRACsiteName = name.split('@').pop()
else:
DIRACsiteName = name
gridSiteName = getGOCSiteName(DIRACsiteName)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'Site_Panel':
gran = 'Site'
paramsL.insert(0, 'SiteName')
siteName = name
elif what == 'Service_Computing_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_Storage_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VO-BOX_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VOMS_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Resource_Panel':
gran = 'Resource'
paramsL.insert(0, 'ResourceName')
resourceName = name
elif what == 'SE_Panel':
gran = 'StorageElement'
paramsL.insert(0, 'StorageElementName')
storageElementName = name
info_bit_got = self.rsDB.getMonitoredsList(
gran,
paramsList=paramsL,
siteName=siteName,
serviceName=serviceName,
serviceType=serviceType,
resourceName=resourceName,
storageElementName=storageElementName,
gridSiteName=gridSiteName)
return info_bit_got
#############################################################################
def _getPolicyDesc(self, policyName):
return self.configModule.Policies[policyName]['Description']
#############################################################################
def __getNameForPanel(self, granularity, name, panel):
if granularity in ('Site', 'Sites'):
if panel == 'Service_Computing_Panel':
granularity = 'Service'
name = 'Computing@' + name
elif panel == 'Service_Storage_Panel':
granularity = 'Service'
name = 'Storage@' + name
elif panel == 'OtherServices_Panel':
granularity = 'Service'
name = 'OtherS@' + name
elif panel == 'Service_VOMS_Panel':
granularity = 'Service'
name = 'VOMS@' + name
elif panel == 'Service_VO-BOX_Panel':
granularity = 'Service'
name = 'VO-BOX@' + name
# else:
# granularity = granularity
# name = name
# else:
# granularity = granularity
# name = name
return (granularity, name)
#############################################################################
def _resExist(self, granularity, name):
siteName = None
serviceName = None
resourceName = None
storageElementName = None
if granularity in ('Site', 'Sites'):
siteName = name
elif granularity in ('Service', 'Services'):
serviceName = name
elif granularity in ('Resource', 'Resources'):
resourceName = name
elif granularity in ('StorageElement', 'StorageElements'):
storageElementName = name
res = self.rsDB.getMonitoredsList(
granularity,
siteName=siteName,
serviceName=serviceName,
resourceName=resourceName,
storageElementName=storageElementName)
if res == []:
return False
else:
return True
class PolicyCaller:
#############################################################################
def __init__(self, commandCallerIn=None):
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
self.policyInvoker = PolicyInvoker()
#############################################################################
def policyInvocation(self,
VOExtension,
granularity=None,
name=None,
status=None,
policy=None,
args=None,
pName=None,
pModule=None,
extraArgs=None,
commandIn=None):
"""
Invokes a policy:
1. If :attr:`policy` is not None, import the right policy module,
specified with :attr:`VOExtension` (e.g.: 'LHCb') and
:attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
"""
p = policy
a = args
moduleBase = VOExtension + "DIRAC.ResourceStatusSystem.Policy."
if p is None:
try:
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ['*'])
except ImportError:
pModule = "AlwaysFalse_Policy"
module = moduleBase + pModule
policyModule = __import__(module, globals(), locals(), ['*'])
p = getattr(policyModule, pModule)()
if a is None:
a = (granularity, name)
if extraArgs is not None:
if isinstance(extraArgs, tuple):
a = a + extraArgs
elif isinstance(extraArgs, list):
argsList = []
for argsTuple in extraArgs:
argsList.append(a + argsTuple)
a = argsList
if commandIn is not None:
commandIn = self.cc.setCommandObject(commandIn)
res = self._innerEval(p, a, commandIn=commandIn)
res['PolicyName'] = pName
return res
#############################################################################
def _innerEval(self, p, a, commandIn=None, knownInfo=None):
""" policy evaluation
"""
self.policyInvoker.setPolicy(p)
p.setArgs(a)
p.setCommand(commandIn)
# p.setInfoName('Result')
res = self.policyInvoker.evaluatePolicy()
return res
def initialize( self ):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsList_AccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def initialize(self):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsListAccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
commandsVOBOXAvailability = (
'VOBOXAvailabilityCommand',
'VOBOXAvailabilityCommand',
)
commandsSpaceTokenOccupancy = (
'SpaceTokenOccupancyCommand',
'SpaceTokenOccupancyCommand',
)
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [
'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient'
]
knownAPIs = initAPIs(knownAPIs, {})
for command in commandsListClientsCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
self.commandObjectsListClientsCache.append((command, cObj))
for command in commandsListAccountingCache:
cObj = cc.setCommandObject(command)
for apiName, apiInstance in knownAPIs.items():
cc.setAPI(cObj, apiName, apiInstance)
cArgs = command[2]
self.commandObjectsListAccountingCache.append(
(command, cObj, cArgs))
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject(commandsVOBOXAvailability)
self.commandObjectsVOBOXAvailability.append(
(commandsVOBOXAvailability, cObj, cArgs))
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject(commandsSpaceTokenOccupancy)
self.commandObjectsSpaceTokenOccupancy.append(
(commandsSpaceTokenOccupancy, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
def initialize(self):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsList_AccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
RPCAccounting = RPCClient("Accounting/ReportGenerator")
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command,
cObj,
RPCWMSAdmin=RPCWMSAdmin,
RPCAccounting=RPCAccounting)
self.commandObjectsList_ClientsCache.append((command, cObj))
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append(
(command, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize( self ):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rmClient = ResourceManagementClient()
self.clientsInvoker = ClientsInvoker()
commandsListClientsCache = [
( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
( 'ClientsCache_Command', 'DTEverySites_Command' ),
( 'ClientsCache_Command', 'DTEveryResources_Command' )
]
commandsListAccountingCache = [
( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
]
commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', )
commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', )
self.commandObjectsListClientsCache = []
self.commandObjectsListAccountingCache = []
self.commandObjectsVOBOXAvailability = []
self.commandObjectsSpaceTokenOccupancy = []
cc = CommandCaller()
# We know beforehand which APIs are we going to need, so we initialize them
# first, making everything faster.
knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
knownAPIs = initAPIs( knownAPIs, {} )
for command in commandsListClientsCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
self.commandObjectsListClientsCache.append( ( command, cObj ) )
for command in commandsListAccountingCache:
cObj = cc.setCommandObject( command )
for apiName, apiInstance in knownAPIs.items():
cc.setAPI( cObj, apiName, apiInstance )
cArgs = command[ 2 ]
self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) )
for cArgs in self.__getVOBOXAvailabilityCandidates():
cObj = cc.setCommandObject( commandsVOBOXAvailability )
self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) )
for cArgs in self.__getSpaceTokenOccupancyCandidates():
cObj = cc.setCommandObject( commandsSpaceTokenOccupancy )
self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "CacheFeederAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )