def setUp(self): # Create a mock of DB class self.mock_DB = Mock() # Setting mock return value self.mock_DB._query.return_value = {'OK': True, 'Value': ''} self.mock_DB._update.return_value = {'OK': True, 'Value': ''} sys.modules["DIRAC"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.ResourceStatusSystem.Utilities.CS"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Core.Utilities.SiteCEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Core.Utilities.SiteSEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Core.Utilities.SitesDIRACGOCDBmapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger from LHCbDIRAC.ResourceStatusSystem.Policy import Configurations #from DIRAC.ResourceStatusSystem.DB.test.FakeResourceStatusDB import FakeResourceStatusDB from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB # setting mock interface self.rsDB = ResourceStatusDB(DBin=self.mock_DB) self.mock_DB_1 = Mock() self.mock_DB_1._query.return_value = { 'OK': True, 'Value': (('VOMS', ), ) } self.rsDB_1 = ResourceStatusDB(DBin=self.mock_DB_1)
def initializeResourceStatusHandler(_serviceInfo): ''' Handler initialization, where we set the ResourceStatusDB as global db, and we instantiate the synchronizer. ''' global db db = ResourceStatusDB() # Publisher is on boxes right now # # rmDB = ResourceStatusDB() # cc = CommandCaller() # global VOExtension # VOExtension = getExt() # ig = InfoGetter( VOExtension ) # WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" ) # global publisher # publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc, # infoGetterIn = ig, WMSAdminIn = WMSAdmin ) syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer') syncObject = syncModule.Synchronizer() gConfig.addListenerToNewVersionEvent(syncObject.sync) return S_OK()
def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport( "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {}
def initialize( self ): """ Standard constructor """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.StorageElementToBeChecked = Queue.Queue() self.StorageElementInCheck = [] self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 ) self.threadPool = ThreadPool( self.maxNumberOfThreads, self.maxNumberOfThreads ) if not self.threadPool: self.log.error( 'Can not create Thread Pool' ) return S_ERROR( 'Can not create Thread Pool' ) self.setup = getSetup()[ 'Value' ] self.VOExtension = getExt() self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ] self.nc = NotificationClient() self.diracAdmin = DiracAdmin() self.csAPI = CSAPI() for _i in xrange( self.maxNumberOfThreads ): self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) ) return S_OK() except Exception: errorStr = "StElWriteInspectorAgent initialization" gLogger.exception( errorStr ) return S_ERROR( errorStr )
def initializeResourceManagementHandler(serviceInfo): global rsDB rsDB = ResourceStatusDB() global rmDB rmDB = ResourceManagementDB() cc = CommandCaller() global VOExtension VOExtension = getExt() ig = InfoGetter(VOExtension) WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") global publisher publisher = Publisher(VOExtension, rsDBIn=rsDB, commandCallerIn=cc, infoGetterIn=ig, WMSAdminIn=WMSAdmin) # sync_O = Synchronizer(rsDB) # gConfig.addListenerToNewVersionEvent( sync_O.sync ) return S_OK()
def __init__(self, rsDBin=None): self.rsDB = rsDBin if self.rsDB == None: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() self.GOCDBClient = GOCDBClient()
def initializeResourceStatusHandler(_serviceInfo): ''' Handler initialization, where we set the ResourceStatusDB as global db, and we instantiate the synchronizer. ''' global db db = ResourceStatusDB() return S_OK()
def initializeResourceStatusHandler( _serviceInfo ): ''' Handler initialization, where we set the ResourceStatusDB as global db, and we instantiate the synchronizer. ''' global db db = ResourceStatusDB() # Regenerates DB tables if needed db._checkTable() return S_OK()
def initialize(self): """ TokenAgent initialization """ try: self.rsDB = ResourceStatusDB() self.nc = NotificationClient() return S_OK() except Exception: errorStr = "TokenAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def initialize( self ): """ ClientsCacheFeeder initialization """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.clientsInvoker = ClientsInvoker() VOExtension = getExt() configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*'] ) commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache ) commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache ) self.commandObjectsList_ClientsCache = [] self.commandObjectsList_AccountingCache = [] cc = CommandCaller() RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" ) RPCAccounting = RPCClient( "Accounting/ReportGenerator" ) for command in commandsList_ClientsCache: cObj = cc.setCommandObject( command ) cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin, RPCAccounting = RPCAccounting ) self.commandObjectsList_ClientsCache.append( ( command, cObj ) ) for command in commandsList_AccountingCache: cObj = cc.setCommandObject( command ) cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting ) try: cArgs = command[2] except IndexError: cArgs = () self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) ) return S_OK() except Exception: errorStr = "ClientsCacheFeeder initialization" gLogger.exception( errorStr ) return S_ERROR( errorStr )
def initialize(self): """ Standard constructor """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.ResourcesToBeChecked = Queue.Queue() self.ResourceNamesInCheck = [] self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1) self.threadPool = ThreadPool(self.maxNumberOfThreads, self.maxNumberOfThreads) if not self.threadPool: self.log.error('Can not create Thread Pool') return S_ERROR('Can not create Thread Pool') self.setup = getSetup()['Value'] self.VOExtension = getExt() configModule = __import__( self.VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*']) self.Resources_check_freq = copy.deepcopy( configModule.Resources_check_freq) self.nc = NotificationClient() self.diracAdmin = DiracAdmin() self.csAPI = CSAPI() for i in xrange(self.maxNumberOfThreads): self.threadPool.generateJobAndQueueIt(self._executeCheck, args=(None, )) return S_OK() except Exception: errorStr = "RSInspectorAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def initialize(self): """ TokenAgent initialization """ self.ELEMENTS = ['Site', 'StorageElementRead', 'StorageElementWrite'] self.notifyHours = self.am_getOption('notifyHours', 10) try: self.rsDB = ResourceStatusDB() self.nc = NotificationClient() self.VOExt = getExt() return S_OK() except Exception: errorStr = "TokenAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def initialize(self): """ CleanerAgent initialization """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.tablesWithHistory = self.rsDB.getTablesWithHistory() self.historyTables = [ x + 'History' for x in self.tablesWithHistory ] return S_OK() except Exception: errorStr = "CleanerAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def installDB(): ''' Installs Tables. ''' db1 = ResourceStatusDB() db2 = ResourceManagementDB() res1 = db1._checkTable() if not res1['OK']: subLogger.error(res1['Message']) elif res1['Value']: subLogger.info('ResourceStatusDB') subLogger.info(res1['Value']) res2 = db2._checkTable() if not res2['OK']: subLogger.error(res2['Message']) elif res2['Value']: subLogger.info('ResourceManagementDB') subLogger.info(res2['Value']) return S_OK()
def enforce(self, pdpIn = None, rsDBIn = None, rmDBIn = None, ncIn = None, setupIn = None, daIn = None, csAPIIn = None, knownInfo = None): """ enforce policies, using a PDP (Policy Decision Point), based on self.__granularity (optional) self.__name (optional) self.__status (optional) self.__formerStatus (optional) self.__reason (optional) self.__siteType (optional) self.__serviceType (optional) self.__realBan (optional) self.__user (optional) self.__futurePolicyType (optional) self.__futureGranularity (optional) :params: :attr:`pdpIn`: a custom PDP object (optional) :attr:`rsDBIn`: a custom (statuses) database object (optional) :attr:`rmDBIn`: a custom (management) database object (optional) :attr:`setupIn`: a string with the present setup (optional) :attr:`ncIn`: a custom notification client object (optional) :attr:`daIn`: a custom DiracAdmin object (optional) :attr:`csAPIIn`: a custom CSAPI object (optional) :attr:`knownInfo`: a string of known provided information (optional) """ #PDP if pdpIn is not None: pdp = pdpIn else: # Use standard DIRAC PDP from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP pdp = PDP(self.VOExtension, granularity = self.__granularity, name = self.__name, status = self.__status, formerStatus = self.__formerStatus, reason = self.__reason, siteType = self.__siteType, serviceType = self.__serviceType, resourceType = self.__resourceType, useNewRes = self.useNewRes) #DB if rsDBIn is not None: rsDB = rsDBIn else: # Use standard DIRAC DB from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB rsDB = ResourceStatusDB() if rmDBIn is not None: rmDB = rmDBIn else: # Use standard DIRAC DB from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB rmDB = ResourceManagementDB() #setup if setupIn is not None: setup = setupIn else: # get present setup setup = getSetup()['Value'] #notification client if ncIn is not None: nc = ncIn else: from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient nc = NotificationClient() #DiracAdmin if daIn is not None: da = daIn else: from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin da = DiracAdmin() #CSAPI if csAPIIn is not None: csAPI = csAPIIn else: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI csAPI = CSAPI() # policy decision resDecisions = pdp.takeDecision(knownInfo=knownInfo) # if self.__name == 'CERN-RAW': # print resDecisions for res in resDecisions['PolicyCombinedResult']: self.__policyType = res['PolicyType'] #if self.__realBan == False: # continue if 'Resource_PolType' in self.__policyType: # If token != RS_SVC, we do not update the token, just the LastCheckedTime if self.__realBan == False: rsDB.setLastMonitoredCheckTime(self.__granularity, self.__name) else: self._ResourcePolTypeActions(resDecisions, res, rsDB, rmDB) if 'Alarm_PolType' in self.__policyType: self._AlarmPolTypeActions(res, nc, setup, rsDB) if 'RealBan_PolType' in self.__policyType and self.__realBan == True: self._RealBanPolTypeActions(res, da, csAPI, setup) if 'Collective_PolType' in self.__policyType: # do something pass
def initialize(self): """ ClientsCacheFeederAgent initialization """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.clientsInvoker = ClientsInvoker() commandsList_ClientsCache = [ ('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'), ('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'), ('ClientsCache_Command', 'DTEverySites_Command'), ('ClientsCache_Command', 'DTEveryResources_Command') ] commandsList_AccountingCache = [ ('AccountingCache_Command', 'TransferQualityByDestSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'), ] self.commandObjectsList_ClientsCache = [] self.commandObjectsList_AccountingCache = [] cc = CommandCaller() RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") RPCAccounting = RPCClient("Accounting/ReportGenerator") for command in commandsList_ClientsCache: cObj = cc.setCommandObject(command) cc.setCommandClient(command, cObj, RPCWMSAdmin=RPCWMSAdmin, RPCAccounting=RPCAccounting) self.commandObjectsList_ClientsCache.append((command, cObj)) for command in commandsList_AccountingCache: cObj = cc.setCommandObject(command) cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting) try: cArgs = command[2] except IndexError: cArgs = () self.commandObjectsList_AccountingCache.append( (command, cObj, cArgs)) return S_OK() except Exception: errorStr = "ClientsCacheFeederAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def enforce(self, pdpIn=None, rsDBIn=None, rmDBIn=None, ncIn=None, setupIn=None, daIn=None, csAPIIn=None, knownInfo=None): """ enforce policies, using a PDP (Policy Decision Point), based on self.__granularity (optional) self.__name (optional) self.__status (optional) self.__formerStatus (optional) self.__reason (optional) self.__siteType (optional) self.__serviceType (optional) self.__realBan (optional) self.__user (optional) self.__futurePolicyType (optional) self.__futureGranularity (optional) :params: :attr:`pdpIn`: a custom PDP object (optional) :attr:`rsDBIn`: a custom (statuses) database object (optional) :attr:`rmDBIn`: a custom (management) database object (optional) :attr:`setupIn`: a string with the present setup (optional) :attr:`ncIn`: a custom notification client object (optional) :attr:`daIn`: a custom DiracAdmin object (optional) :attr:`csAPIIn`: a custom CSAPI object (optional) :attr:`knownInfo`: a string of known provided information (optional) """ #PDP if pdpIn is not None: pdp = pdpIn else: # Use standard DIRAC PDP from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP pdp = PDP(self.VOExtension, granularity=self.__granularity, name=self.__name, status=self.__status, formerStatus=self.__formerStatus, reason=self.__reason, siteType=self.__siteType, serviceType=self.__serviceType, resourceType=self.__resourceType, useNewRes=self.useNewRes) #DB if rsDBIn is not None: rsDB = rsDBIn else: # Use standard DIRAC DB from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB rsDB = ResourceStatusDB() if rmDBIn is not None: rmDB = rmDBIn else: # Use standard DIRAC DB from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB rmDB = ResourceManagementDB() #setup if setupIn is not None: setup = setupIn else: # get present setup setup = CS.getSetup()['Value'] #notification client if ncIn is not None: nc = ncIn else: from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient nc = NotificationClient() #DiracAdmin if daIn is not None: da = daIn else: from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin da = DiracAdmin() #CSAPI if csAPIIn is not None: csAPI = csAPIIn else: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI csAPI = CSAPI() ################### # policy decision # ################### resDecisions = pdp.takeDecision(knownInfo=knownInfo) assert (type(resDecisions) == dict and resDecisions != {}) res = resDecisions['PolicyCombinedResult'] actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions" # Security mechanism in case there is no PolicyType returned if res == {}: EmptyPolTypeActions(self.__granularity, self.__name, resDecisions, res) else: policyType = res['PolicyType'] if 'Resource_PolType' in policyType: m = Utils.voimport(actionBaseMod + ".Resource_PolType", self.VOExtension) m.ResourcePolTypeActions(self.__granularity, self.__name, resDecisions, res, rsDB, rmDB) if 'Alarm_PolType' in policyType: m = Utils.voimport(actionBaseMod + ".Alarm_PolType", self.VOExtension) m.AlarmPolTypeActions(self.__name, res, nc, setup, rsDB, rmDB, Granularity=self.__granularity, SiteType=self.__siteType, ServiceType=self.__serviceType, ResourceType=self.__resourceType) if 'RealBan_PolType' in policyType and self.__realBan == True: m = Utils.voimport(actionBaseMod + ".RealBan_PolType", self.VOExtension) m.RealBanPolTypeActions(self.__granularity, self.__name, res, da, csAPI, setup)