def evaluate(self): """ Before use, call at least `setArgs` and, alternatively, `setCommand` or `setCommandName`. Invoking `super(PolicyCLASS, self).evaluate` will invoke the command (if necessary) as it is provided and returns the results. """ if self.knownInfo: result = self.knownInfo else: if not self.command: # use standard Command from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller cc = CommandCaller() self.command = cc.setCommandObject(self.commandName) clientsInvoker = ClientsInvoker() clientsInvoker.setCommand(self.command) self.command.setArgs(self.args) result = clientsInvoker.doCommand() if not self.infoName: result = result['Result'] else: if self.infoName in result.keys(): result = result[self.infoName] else: raise RSSException, "missing 'infoName' in result" return result
def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport( "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {}
def __init__(self, commandCallerIn=None): if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() self.policyInvoker = PolicyInvoker()
def initializeResourceManagementHandler(serviceInfo): global rsDB rsDB = ResourceStatusDB() global rmDB rmDB = ResourceManagementDB() cc = CommandCaller() global VOExtension VOExtension = getExt() ig = InfoGetter(VOExtension) WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") global publisher publisher = Publisher(VOExtension, rsDBIn=rsDB, commandCallerIn=cc, infoGetterIn=ig, WMSAdminIn=WMSAdmin) # sync_O = Synchronizer(rsDB) # gConfig.addListenerToNewVersionEvent( sync_O.sync ) return S_OK()
def __init__(self, commandCallerIn=None, **clients): ''' Constructor ''' if commandCallerIn is None: commandCallerIn = CommandCaller() self.cCaller = commandCallerIn self.clients = clients
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None, WMSAdminIn = None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool( 2, 5 ) self.lockObj = threading.RLock() self.infoForPanel_res = {}
def initialize( self ): """ ClientsCacheFeeder initialization """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.clientsInvoker = ClientsInvoker() VOExtension = getExt() configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*'] ) commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache ) commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache ) self.commandObjectsList_ClientsCache = [] self.commandObjectsList_AccountingCache = [] cc = CommandCaller() RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" ) RPCAccounting = RPCClient( "Accounting/ReportGenerator" ) for command in commandsList_ClientsCache: cObj = cc.setCommandObject( command ) cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin, RPCAccounting = RPCAccounting ) self.commandObjectsList_ClientsCache.append( ( command, cObj ) ) for command in commandsList_AccountingCache: cObj = cc.setCommandObject( command ) cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting ) try: cArgs = command[2] except IndexError: cArgs = () self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) ) return S_OK() except Exception: errorStr = "ClientsCacheFeeder initialization" gLogger.exception( errorStr ) return S_ERROR( errorStr )
def evaluate(self): ''' Before use, call at least `setArgs` and, alternatively, `setCommand` or `setCommandName`. Invoking `super(PolicyCLASS, self).evaluate` will invoke the command (if necessary) as it is provided and returns the results. ''' if self.knownInfo: result = self.knownInfo else: if not self.command: # use standard Command cc = CommandCaller() self.command = cc.setCommandObject(self.commandName) clientsInvoker = ClientsInvoker() clientsInvoker.setCommand(self.command) self.command.setArgs(self.args) result = clientsInvoker.doCommand() if not self.infoName: result = result['Result'] else: if self.infoName in result.keys(): result = result[self.infoName] else: gLogger.error('missing "infoName" in result') return None return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def evaluate( self ): ''' Before use, call at least `setArgs` and, alternatively, `setCommand` or `setCommandName`. Invoking `super(PolicyCLASS, self).evaluate` will invoke the command (if necessary) as it is provided and returns the results. ''' if self.knownInfo: result = self.knownInfo else: if not self.command: # use standard Command cc = CommandCaller() self.command = cc.setCommandObject( self.commandName ) clientsInvoker = ClientsInvoker() clientsInvoker.setCommand( self.command ) self.command.setArgs( self.args ) result = clientsInvoker.doCommand() if not self.infoName: result = result[ 'Result' ] else: if self.infoName in result.keys(): result = result[ self.infoName ] else: gLogger.error( 'missing "infoName" in result' ) return None return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__(self, **clients): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller(cc, **clients) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None
def __init__(self, VOExtension, granularity=None, name=None, status=None, formerStatus=None, reason=None, siteType=None, serviceType=None, resourceType=None, useNewRes=False): """ PDP (Policy Decision Point) initialization :params: :attr:`VOExtension`: string - VO extension (e.g. 'LHCb') :attr:`granularity`: string - a ValidRes :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.VOExtension = VOExtension self.__granularity = assignOrRaise(granularity, ValidRes, InvalidRes, self, self.__init__) self.__name = name self.__status = assignOrRaise(status, ValidStatus, InvalidStatus, self, self.__init__) self.__formerStatus = assignOrRaise(formerStatus, ValidStatus, InvalidStatus, self, self.__init__) self.__reason = reason self.__siteType = assignOrRaise(siteType, ValidSiteType, InvalidSiteType, self, self.__init__) self.__serviceType = assignOrRaise(serviceType, ValidServiceType, InvalidServiceType, self, self.__init__) self.__resourceType = assignOrRaise(resourceType, ValidResourceType, InvalidResourceType, self, self.__init__) cc = CommandCaller() self.pc = PolicyCaller(cc) self.useNewRes = useNewRes self.args = None self.policy = None self.knownInfo = None self.ig = None
class PolicyCaller: ############################################################################# def __init__(self, commandCallerIn=None): if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() self.policyInvoker = PolicyInvoker() ############################################################################# def policyInvocation( self, VOExtension, granularity=None, name=None, status=None, policy=None, args=None, pName=None, pModule=None, extraArgs=None, commandIn=None, ): """ Invokes a policy: 1. If :attr:`policy` is not None, import the right policy module, specified with :attr:`VOExtension` (e.g.: 'LHCb') and :attr:`pModule` (e.g. 'DT_Policy'). 1.1. Create a policy object. 2. Set the policy arguments (usually :attr:`granularity`, :attr:`name`) + :attr:`extraArgs`. 3. If commandIn is specified (normally it is), use :meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject` to get a command object """ p = policy a = args moduleBase = VOExtension + "DIRAC.ResourceStatusSystem.Policy." if p is None: try: module = moduleBase + pModule policyModule = __import__(module, globals(), locals(), ["*"]) except ImportError: pModule = "AlwaysFalse_Policy" module = moduleBase + pModule policyModule = __import__(module, globals(), locals(), ["*"]) p = getattr(policyModule, pModule)() if a is None: a = (granularity, name) if extraArgs is not None: if isinstance(extraArgs, tuple): a = a + extraArgs elif isinstance(extraArgs, list): argsList = [] for argsTuple in extraArgs: argsList.append(a + argsTuple) a = argsList if commandIn is not None: commandIn = self.cc.setCommandObject(commandIn) res = self._innerEval(p, a, commandIn=commandIn) res["PolicyName"] = pName return res ############################################################################# def _innerEval(self, p, a, commandIn=None, knownInfo=None): """ policy evaluation """ self.policyInvoker.setPolicy(p) p.setArgs(a) p.setCommand(commandIn) # p.setInfoName('Result') res = self.policyInvoker.evaluatePolicy() return res
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None, WMSAdminIn = None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool( 2, 5 ) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes = False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, Utils.where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'], resourceName = name)[0][0] except IndexError: return "%s does not exist!" %name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'], storageElementName = name)[0][0] except IndexError: return "%s does not exist!" %name paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description'] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status, formerStatus = formerStatus, siteType = siteType, serviceType = serviceType, resourceType = resourceType, useNewRes = useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt(self.getInfoForPanel, args = (info, granularityForPanel, nameForPanel) ) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]} else: pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format_ = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what) info_res[format_] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, Utils.where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = { name : { 'RSSStatus': RSSStatus} } return status ############################################################################# def _getInfo(self, granularity, name, format_, what): if format_ == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName, serviceName = serviceName, serviceType = serviceType, resourceName = resourceName, storageElementName = storageElementName, gridSiteName = gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList(granularity, siteName = siteName, serviceName = serviceName, resourceName = resourceName, storageElementName = storageElementName) if res == []: return False else: return True
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*']) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes=False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList( 'Resource', ['ResourceType'], resourceName=name)[0][0] except IndexError: return "%s does not exist!" % name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList( 'StorageElement', ['SiteType'], storageElementName=name)[0][0] except IndexError: return "%s does not exist!" % name paramNames = [ 'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description' ] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status=status, formerStatus=formerStatus, siteType=siteType, serviceType=serviceType, resourceType=resourceType, useNewRes=useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel][ 'DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel][ 'RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt( self.getInfoForPanel, args=(info, granularityForPanel, nameForPanel)) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy][ 'Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]} else: pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format, what) info_res[format] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name: { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = {name: {'RSSStatus': RSSStatus}} return status ############################################################################# def _getInfo(self, granularity, name, format, what): if format == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList( gran, paramsList=paramsL, siteName=siteName, serviceName=serviceName, serviceType=serviceType, resourceName=resourceName, storageElementName=storageElementName, gridSiteName=gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList( granularity, siteName=siteName, serviceName=serviceName, resourceName=resourceName, storageElementName=storageElementName) if res == []: return False else: return True
class PolicyCaller: ############################################################################# def __init__(self, commandCallerIn=None): if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() self.policyInvoker = PolicyInvoker() ############################################################################# def policyInvocation(self, VOExtension, granularity=None, name=None, status=None, policy=None, args=None, pName=None, pModule=None, extraArgs=None, commandIn=None): """ Invokes a policy: 1. If :attr:`policy` is not None, import the right policy module, specified with :attr:`VOExtension` (e.g.: 'LHCb') and :attr:`pModule` (e.g. 'DT_Policy'). 1.1. Create a policy object. 2. Set the policy arguments (usually :attr:`granularity`, :attr:`name`) + :attr:`extraArgs`. 3. If commandIn is specified (normally it is), use :meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject` to get a command object """ p = policy a = args moduleBase = VOExtension + "DIRAC.ResourceStatusSystem.Policy." if p is None: try: module = moduleBase + pModule policyModule = __import__(module, globals(), locals(), ['*']) except ImportError: pModule = "AlwaysFalse_Policy" module = moduleBase + pModule policyModule = __import__(module, globals(), locals(), ['*']) p = getattr(policyModule, pModule)() if a is None: a = (granularity, name) if extraArgs is not None: if isinstance(extraArgs, tuple): a = a + extraArgs elif isinstance(extraArgs, list): argsList = [] for argsTuple in extraArgs: argsList.append(a + argsTuple) a = argsList if commandIn is not None: commandIn = self.cc.setCommandObject(commandIn) res = self._innerEval(p, a, commandIn=commandIn) res['PolicyName'] = pName return res ############################################################################# def _innerEval(self, p, a, commandIn=None, knownInfo=None): """ policy evaluation """ self.policyInvoker.setPolicy(p) p.setArgs(a) p.setCommand(commandIn) # p.setInfoName('Result') res = self.policyInvoker.evaluatePolicy() return res
def initialize( self ): """ ClientsCacheFeederAgent initialization """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.clientsInvoker = ClientsInvoker() commandsList_ClientsCache = [ ( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ), ( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ), ( 'ClientsCache_Command', 'DTEverySites_Command' ), ( 'ClientsCache_Command', 'DTEveryResources_Command' ) ] commandsList_AccountingCache = [ ( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ), ( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ), ( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ), ] self.commandObjectsList_ClientsCache = [] self.commandObjectsList_AccountingCache = [] cc = CommandCaller() RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" ) RPCAccounting = RPCClient( "Accounting/ReportGenerator" ) for command in commandsList_ClientsCache: cObj = cc.setCommandObject( command ) cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin, RPCAccounting = RPCAccounting ) self.commandObjectsList_ClientsCache.append( ( command, cObj ) ) for command in commandsList_AccountingCache: cObj = cc.setCommandObject( command ) cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting ) try: cArgs = command[2] except IndexError: cArgs = () self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) ) return S_OK() except Exception: errorStr = "ClientsCacheFeederAgent initialization" gLogger.exception( errorStr ) return S_ERROR( errorStr )
def initialize(self): # Attribute defined outside __init__ # pylint: disable-msg=W0201 try: self.rmClient = ResourceManagementClient() self.clientsInvoker = ClientsInvoker() commandsListClientsCache = [ ('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'), ('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'), ('ClientsCache_Command', 'DTEverySites_Command'), ('ClientsCache_Command', 'DTEveryResources_Command') ] commandsListAccountingCache = [ ('AccountingCache_Command', 'TransferQualityByDestSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'), ] commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', ) commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', ) self.commandObjectsListClientsCache = [] self.commandObjectsListAccountingCache = [] self.commandObjectsVOBOXAvailability = [] self.commandObjectsSpaceTokenOccupancy = [] cc = CommandCaller() # We know beforehand which APIs are we going to need, so we initialize them # first, making everything faster. knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator', 'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ] knownAPIs = initAPIs(knownAPIs, {}) for command in commandsListClientsCache: cObj = cc.setCommandObject(command) for apiName, apiInstance in knownAPIs.items(): cc.setAPI(cObj, apiName, apiInstance) self.commandObjectsListClientsCache.append((command, cObj)) for command in commandsListAccountingCache: cObj = cc.setCommandObject(command) for apiName, apiInstance in knownAPIs.items(): cc.setAPI(cObj, apiName, apiInstance) cArgs = command[2] self.commandObjectsListAccountingCache.append( (command, cObj, cArgs)) for cArgs in self.__getVOBOXAvailabilityCandidates(): cObj = cc.setCommandObject(commandsVOBOXAvailability) self.commandObjectsVOBOXAvailability.append( (commandsVOBOXAvailability, cObj, cArgs)) for cArgs in self.__getSpaceTokenOccupancyCandidates(): cObj = cc.setCommandObject(commandsSpaceTokenOccupancy) self.commandObjectsSpaceTokenOccupancy.append( (commandsSpaceTokenOccupancy, cObj, cArgs)) return S_OK() except Exception: errorStr = "CacheFeederAgent initialization" self.log.exception(errorStr) return S_ERROR(errorStr)
def initialize(self): """ ClientsCacheFeederAgent initialization """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.clientsInvoker = ClientsInvoker() commandsList_ClientsCache = [ ('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'), ('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'), ('ClientsCache_Command', 'DTEverySites_Command'), ('ClientsCache_Command', 'DTEveryResources_Command') ] commandsList_AccountingCache = [ ('AccountingCache_Command', 'TransferQualityByDestSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'), ] self.commandObjectsList_ClientsCache = [] self.commandObjectsList_AccountingCache = [] cc = CommandCaller() RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") RPCAccounting = RPCClient("Accounting/ReportGenerator") for command in commandsList_ClientsCache: cObj = cc.setCommandObject(command) cc.setCommandClient(command, cObj, RPCWMSAdmin=RPCWMSAdmin, RPCAccounting=RPCAccounting) self.commandObjectsList_ClientsCache.append((command, cObj)) for command in commandsList_AccountingCache: cObj = cc.setCommandObject(command) cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting) try: cArgs = command[2] except IndexError: cArgs = () self.commandObjectsList_AccountingCache.append( (command, cObj, cArgs)) return S_OK() except Exception: errorStr = "ClientsCacheFeederAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def initialize( self ): # Attribute defined outside __init__ # pylint: disable-msg=W0201 try: self.rmClient = ResourceManagementClient() self.clientsInvoker = ClientsInvoker() commandsListClientsCache = [ ( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ), ( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ), ( 'ClientsCache_Command', 'DTEverySites_Command' ), ( 'ClientsCache_Command', 'DTEveryResources_Command' ) ] commandsListAccountingCache = [ ( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ), ( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ), ( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ), ] commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', ) commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', ) self.commandObjectsListClientsCache = [] self.commandObjectsListAccountingCache = [] self.commandObjectsVOBOXAvailability = [] self.commandObjectsSpaceTokenOccupancy = [] cc = CommandCaller() # We know beforehand which APIs are we going to need, so we initialize them # first, making everything faster. knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator', 'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ] knownAPIs = initAPIs( knownAPIs, {} ) for command in commandsListClientsCache: cObj = cc.setCommandObject( command ) for apiName, apiInstance in knownAPIs.items(): cc.setAPI( cObj, apiName, apiInstance ) self.commandObjectsListClientsCache.append( ( command, cObj ) ) for command in commandsListAccountingCache: cObj = cc.setCommandObject( command ) for apiName, apiInstance in knownAPIs.items(): cc.setAPI( cObj, apiName, apiInstance ) cArgs = command[ 2 ] self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) ) for cArgs in self.__getVOBOXAvailabilityCandidates(): cObj = cc.setCommandObject( commandsVOBOXAvailability ) self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) ) for cArgs in self.__getSpaceTokenOccupancyCandidates(): cObj = cc.setCommandObject( commandsSpaceTokenOccupancy ) self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) ) return S_OK() except Exception: errorStr = "CacheFeederAgent initialization" self.log.exception( errorStr ) return S_ERROR( errorStr )