def setUp(self):
# Create a mock of DB class
self.mock_DB = Mock()
# Setting mock return value
self.mock_DB._query.return_value = {'OK': True, 'Value': ''}
self.mock_DB._update.return_value = {'OK': True, 'Value': ''}
sys.modules["DIRAC"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.ResourceStatusSystem.Utilities.CS"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Core.Utilities.SiteCEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Core.Utilities.SiteSEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Core.Utilities.SitesDIRACGOCDBmapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
from LHCbDIRAC.ResourceStatusSystem.Policy import Configurations
#from DIRAC.ResourceStatusSystem.DB.test.FakeResourceStatusDB import FakeResourceStatusDB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
# setting mock interface
self.rsDB = ResourceStatusDB(DBin=self.mock_DB)
self.mock_DB_1 = Mock()
self.mock_DB_1._query.return_value = {
'OK': True,
'Value': (('VOMS', ), )
}
self.rsDB_1 = ResourceStatusDB(DBin=self.mock_DB_1)
def initializeResourceStatusHandler(_serviceInfo):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
# Publisher is on boxes right now
#
# rmDB = ResourceStatusDB()
# cc = CommandCaller()
# global VOExtension
# VOExtension = getExt()
# ig = InfoGetter( VOExtension )
# WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
# global publisher
# publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
# infoGetterIn = ig, WMSAdminIn = WMSAdmin )
syncModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Utilities.Synchronizer')
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent(syncObject.sync)
return S_OK()
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def initialize( self ):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.StorageElementToBeChecked = Queue.Queue()
self.StorageElementInCheck = []
self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
self.threadPool = ThreadPool( self.maxNumberOfThreads,
self.maxNumberOfThreads )
if not self.threadPool:
self.log.error( 'Can not create Thread Pool' )
return S_ERROR( 'Can not create Thread Pool' )
self.setup = getSetup()[ 'Value' ]
self.VOExtension = getExt()
self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ]
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for _i in xrange( self.maxNumberOfThreads ):
self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) )
return S_OK()
except Exception:
errorStr = "StElWriteInspectorAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def initializeResourceManagementHandler(serviceInfo):
global rsDB
rsDB = ResourceStatusDB()
global rmDB
rmDB = ResourceManagementDB()
cc = CommandCaller()
global VOExtension
VOExtension = getExt()
ig = InfoGetter(VOExtension)
WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
global publisher
publisher = Publisher(VOExtension,
rsDBIn=rsDB,
commandCallerIn=cc,
infoGetterIn=ig,
WMSAdminIn=WMSAdmin)
# sync_O = Synchronizer(rsDB)
# gConfig.addListenerToNewVersionEvent( sync_O.sync )
return S_OK()
def __init__(self, rsDBin=None):
self.rsDB = rsDBin
if self.rsDB == None:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
self.GOCDBClient = GOCDBClient()
def initializeResourceStatusHandler(_serviceInfo):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
return S_OK()
def initializeResourceStatusHandler( _serviceInfo ):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
# Regenerates DB tables if needed
db._checkTable()
return S_OK()
def initialize(self):
"""
TokenAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.nc = NotificationClient()
return S_OK()
except Exception:
errorStr = "TokenAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize( self ):
""" ClientsCacheFeeder initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
VOExtension = getExt()
configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'] )
commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )
commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
RPCAccounting = RPCClient( "Accounting/ReportGenerator" )
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
RPCAccounting = RPCAccounting )
self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject( command )
cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
return S_OK()
except Exception:
errorStr = "ClientsCacheFeeder initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def initialize(self):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.ResourcesToBeChecked = Queue.Queue()
self.ResourceNamesInCheck = []
self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
self.threadPool = ThreadPool(self.maxNumberOfThreads,
self.maxNumberOfThreads)
if not self.threadPool:
self.log.error('Can not create Thread Pool')
return S_ERROR('Can not create Thread Pool')
self.setup = getSetup()['Value']
self.VOExtension = getExt()
configModule = __import__(
self.VOExtension +
"DIRAC.ResourceStatusSystem.Policy.Configurations", globals(),
locals(), ['*'])
self.Resources_check_freq = copy.deepcopy(
configModule.Resources_check_freq)
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for i in xrange(self.maxNumberOfThreads):
self.threadPool.generateJobAndQueueIt(self._executeCheck,
args=(None, ))
return S_OK()
except Exception:
errorStr = "RSInspectorAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize(self):
"""
TokenAgent initialization
"""
self.ELEMENTS = ['Site', 'StorageElementRead', 'StorageElementWrite']
self.notifyHours = self.am_getOption('notifyHours', 10)
try:
self.rsDB = ResourceStatusDB()
self.nc = NotificationClient()
self.VOExt = getExt()
return S_OK()
except Exception:
errorStr = "TokenAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize(self):
""" CleanerAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.tablesWithHistory = self.rsDB.getTablesWithHistory()
self.historyTables = [
x + 'History' for x in self.tablesWithHistory
]
return S_OK()
except Exception:
errorStr = "CleanerAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def installDB():
'''
Installs Tables.
'''
db1 = ResourceStatusDB()
db2 = ResourceManagementDB()
res1 = db1._checkTable()
if not res1['OK']:
subLogger.error(res1['Message'])
elif res1['Value']:
subLogger.info('ResourceStatusDB')
subLogger.info(res1['Value'])
res2 = db2._checkTable()
if not res2['OK']:
subLogger.error(res2['Message'])
elif res2['Value']:
subLogger.info('ResourceManagementDB')
subLogger.info(res2['Value'])
return S_OK()
def enforce(self, pdpIn = None, rsDBIn = None, rmDBIn = None, ncIn = None, setupIn = None,
daIn = None, csAPIIn = None, knownInfo = None):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP(self.VOExtension, granularity = self.__granularity, name = self.__name,
status = self.__status, formerStatus = self.__formerStatus, reason = self.__reason,
siteType = self.__siteType, serviceType = self.__serviceType,
resourceType = self.__resourceType, useNewRes = self.useNewRes)
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = getSetup()['Value']
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
# policy decision
resDecisions = pdp.takeDecision(knownInfo=knownInfo)
# if self.__name == 'CERN-RAW':
# print resDecisions
for res in resDecisions['PolicyCombinedResult']:
self.__policyType = res['PolicyType']
#if self.__realBan == False:
# continue
if 'Resource_PolType' in self.__policyType:
# If token != RS_SVC, we do not update the token, just the LastCheckedTime
if self.__realBan == False:
rsDB.setLastMonitoredCheckTime(self.__granularity, self.__name)
else:
self._ResourcePolTypeActions(resDecisions, res, rsDB, rmDB)
if 'Alarm_PolType' in self.__policyType:
self._AlarmPolTypeActions(res, nc, setup, rsDB)
if 'RealBan_PolType' in self.__policyType and self.__realBan == True:
self._RealBanPolTypeActions(res, da, csAPI, setup)
if 'Collective_PolType' in self.__policyType:
# do something
pass
def initialize(self):
""" ClientsCacheFeederAgent initialization
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.clientsInvoker = ClientsInvoker()
commandsList_ClientsCache = [
('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
('ClientsCache_Command', 'DTEverySites_Command'),
('ClientsCache_Command', 'DTEveryResources_Command')
]
commandsList_AccountingCache = [
('AccountingCache_Command',
'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
('AccountingCache_Command',
'TransferQualityByDestSplittedSite_Command', (24, ),
'Hourly'),
('AccountingCache_Command',
'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
(24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
('AccountingCache_Command',
'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
]
self.commandObjectsList_ClientsCache = []
self.commandObjectsList_AccountingCache = []
cc = CommandCaller()
RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
RPCAccounting = RPCClient("Accounting/ReportGenerator")
for command in commandsList_ClientsCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command,
cObj,
RPCWMSAdmin=RPCWMSAdmin,
RPCAccounting=RPCAccounting)
self.commandObjectsList_ClientsCache.append((command, cObj))
for command in commandsList_AccountingCache:
cObj = cc.setCommandObject(command)
cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
try:
cArgs = command[2]
except IndexError:
cArgs = ()
self.commandObjectsList_AccountingCache.append(
(command, cObj, cArgs))
return S_OK()
except Exception:
errorStr = "ClientsCacheFeederAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def enforce(self,
pdpIn=None,
rsDBIn=None,
rmDBIn=None,
ncIn=None,
setupIn=None,
daIn=None,
csAPIIn=None,
knownInfo=None):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP(self.VOExtension,
granularity=self.__granularity,
name=self.__name,
status=self.__status,
formerStatus=self.__formerStatus,
reason=self.__reason,
siteType=self.__siteType,
serviceType=self.__serviceType,
resourceType=self.__resourceType,
useNewRes=self.useNewRes)
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = CS.getSetup()['Value']
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
###################
# policy decision #
###################
resDecisions = pdp.takeDecision(knownInfo=knownInfo)
assert (type(resDecisions) == dict and resDecisions != {})
res = resDecisions['PolicyCombinedResult']
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyPolTypeActions(self.__granularity, self.__name, resDecisions,
res)
else:
policyType = res['PolicyType']
if 'Resource_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Resource_PolType",
self.VOExtension)
m.ResourcePolTypeActions(self.__granularity, self.__name,
resDecisions, res, rsDB, rmDB)
if 'Alarm_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Alarm_PolType",
self.VOExtension)
m.AlarmPolTypeActions(self.__name,
res,
nc,
setup,
rsDB,
rmDB,
Granularity=self.__granularity,
SiteType=self.__siteType,
ServiceType=self.__serviceType,
ResourceType=self.__resourceType)
if 'RealBan_PolType' in policyType and self.__realBan == True:
m = Utils.voimport(actionBaseMod + ".RealBan_PolType",
self.VOExtension)
m.RealBanPolTypeActions(self.__granularity, self.__name, res,
da, csAPI, setup)
