def initializeResourceManagementHandler(serviceInfo):
global rsDB
rsDB = ResourceStatusDB()
global rmDB
rmDB = ResourceManagementDB()
cc = CommandCaller()
global VOExtension
VOExtension = getExt()
ig = InfoGetter(VOExtension)
WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
global publisher
publisher = Publisher(VOExtension,
rsDBIn=rsDB,
commandCallerIn=cc,
infoGetterIn=ig,
WMSAdminIn=WMSAdmin)
# sync_O = Synchronizer(rsDB)
# gConfig.addListenerToNewVersionEvent( sync_O.sync )
return S_OK()
def __init__(self, clients=None):
""" Constructor.
examples:
>>> pdp = PDP( None )
>>> pdp1 = PDP( {} )
>>> pdp2 = PDP( { 'Client1' : Client1Object } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with Clients to be used in the Commands. If None, the Commands
will create their own clients.
"""
# decision parameters used to match policies and actions
self.decisionParams = None
# Helpers to discover policies and RSS metadata in CS
self.iGetter = InfoGetter()
self.pCaller = PolicyCaller(clients)
# RSS State Machine, used to calculate most penalizing state while merging them
self.rssMachine = RSSMachine('Unknown')
self.log = gLogger.getSubLogger('PDP')
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def __init__(self, clients):
'''
Constructor. Defines members that will be used later on.
'''
self.pCaller = PolicyCaller(clients=clients)
self.iGetter = InfoGetter()
self.decissionParams = {}
self.rssMachine = RSSMachine('Unknown')
def setUp(self):
sys.modules["DIRAC"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.ResourceStatusSystem.Utilities.CS"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Core.Utilities.SiteCEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Core.Utilities.SiteSEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Core.Utilities.SitesDIRACGOCDBmapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
sys.modules[
"DIRAC.Interfaces.API.DiracAdmin"] = DIRAC.ResourceStatusSystem.test.fake_Admin
sys.modules[
"DIRAC.FrameworkSystem.Client.NotificationClient"] = DIRAC.ResourceStatusSystem.test.fake_NotificationClient
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
from DIRAC.ResourceStatusSystem.PolicySystem.PolicyBase import PolicyBase
from DIRAC.ResourceStatusSystem.PolicySystem.PolicyInvoker import PolicyInvoker
from DIRAC import gConfig
self.VO = gConfig.getValue("DIRAC/Extensions")
if 'LHCb' in self.VO:
self.VO = 'LHCb'
self.mock_command = Mock()
self.mock_policy = Mock()
self.mock_p = Mock()
self.mock_args = Mock()
self.pb = PolicyBase()
self.pi = PolicyInvoker()
self.mock_pdp = Mock()
self.mock_rsDB = Mock()
self.mock_rmDB = Mock()
self.mock_nc = Mock()
self.mock_da = Mock()
self.mock_da.getBannedSites.return_value = {
'OK': True,
'Value': ['LCG.APC.fr', 'LCG.Bari.it', 'LCG.Catania.it']
}
self.mock_da.addSiteInMask.return_value = {'OK': True, 'Value': ''}
self.mock_da.banSiteFromMask.return_value = {'OK': True, 'Value': ''}
self.mock_da.sendMail.return_value = {'OK': True, 'Value': ''}
self.mock_csAPI = Mock()
self.mock_csAPI.setOption.return_value = {'OK': True, 'Value': ''}
self.mock_csAPI.commit.return_value = {'OK': True, 'Value': ''}
self.ig = InfoGetter(self.VO)
def __init__(self):
"""
Constructor, initializes the rssClient.
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.rssConfig = RssConfiguration()
self.__opHelper = Operations()
self.rssClient = None
self.infoGetter = InfoGetter()
# We can set CacheLifetime and CacheHistory from CS, so that we can tune them.
cacheLifeTime = int(self.rssConfig.getConfigCache())
# RSSCache only affects the calls directed to RSS, if using the CS it is not
# used.
self.seCache = RSSCache('StorageElement', cacheLifeTime,
self.__updateSECache)
def __init__(self, **clients):
'''
Constructor. Defines members that will be used later on.
'''
cc = CommandCaller()
self.clients = clients
self.pCaller = PolicyCaller(cc, **clients)
self.iGetter = InfoGetter()
self.__granularity = None
self.__name = None
self.__statusType = None
self.__status = None
self.__formerStatus = None
self.__reason = None
self.__siteType = None
self.__serviceType = None
self.__resourceType = None
self.__useNewRes = None
def testGetInfoToApply(self):
ig = InfoGetter('LHCb')
for g in ValidRes:
for s in ValidStatus:
for site_t in ValidSiteType:
for service_t in ValidServiceType:
if g in ('Site', 'Sites'):
panel = 'Site_Panel'
if g in ('Service', 'Services'):
if service_t == 'Storage':
panel = 'Service_Storage_Panel'
if service_t == 'Computing':
panel = 'Service_Computing_Panel'
if service_t == 'VO-BOX':
panel = 'Service_VO-BOX_Panel'
if service_t == 'VOMS':
panel = 'Service_VOMS_Panel'
if g in ('Resource', 'Resources'):
panel = 'Resource_Panel'
if g in ('StorageElementRead', 'StorageElementsRead'):
panel = 'SE_Panel'
if g in ('StorageElementWrite', 'StorageElementsWrite'):
panel = 'SE_Panel'
for resource_t in ValidResourceType:
res = ig.getInfoToApply(('policyType', ), g, s, None, site_t, service_t, resource_t)
for p_res in res[0]['PolicyType']:
self.assert_(p_res in ['Resource_PolType', 'Alarm_PolType', 'Alarm_PolType_SE'])
for useNewRes in (False, True):
res = ig.getInfoToApply(('policy', ), g, s, None, site_t, service_t, resource_t, useNewRes)
pModuleList = [None]
for k in self.configModule.Policies.keys():
try:
if self.configModule.Policies[k]['module'] not in pModuleList:
pModuleList.append(self.configModule.Policies[k]['module'])
except KeyError:
pass
for p_res in res[0]['Policies']:
self.assert_(p_res['Name'] in self.configModule.Policies.keys())
self.assert_(p_res['Module'] in pModuleList)
if useNewRes is False:
self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandIn'])
self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['args'])
else:
try:
self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandInNewRes'])
except KeyError:
self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandIn'])
try:
self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['argsNewRes'])
except KeyError:
self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['args'])
res = ig.getInfoToApply(('panel_info', ), g, s, None, site_t, service_t, resource_t, useNewRes)
for p_res in res[0]['Info']:
# if 'JobsEfficiencySimple' in p_res.keys():
# print useNewRes, p_res
for p_name in p_res.keys():
self.assert_(p_name in self.configModule.Policies.keys())
if isinstance(p_res[p_name], list):
for i in range(len(p_res[p_name])):
for k in p_res[p_name][i].keys():
if useNewRes:
try:
self.assertEqual(p_res[p_name][i][k]['CommandIn'],
self.configModule.Policies[p_name][panel][i][k]['CommandInNewRes'])
except KeyError:
self.assertEqual(p_res[p_name][i][k]['CommandIn'],
self.configModule.Policies[p_name][panel][i][k]['CommandIn'])
except TypeError:
self.assertEqual(p_res[p_name][i][k],
self.configModule.Policies[p_name][panel][i][k])
try:
self.assertEqual(p_res[p_name][i][k]['args'],
self.configModule.Policies[p_name][panel][i][k]['argsNewRes'])
except KeyError:
self.assertEqual(p_res[p_name][i][k]['args'],
self.configModule.Policies[p_name][panel][i][k]['args'])
except TypeError:
self.assertEqual(p_res[p_name][i][k],
self.configModule.Policies[p_name][panel][i][k])
else:
try:
self.assertEqual(p_res[p_name][i][k]['CommandIn'],
self.configModule.Policies[p_name][panel][i][k]['CommandIn'])
except:
self.assertEqual(p_res[p_name][i][k],
self.configModule.Policies[p_name][panel][i][k])
try:
self.assertEqual(p_res[p_name][i][k]['args'],
self.configModule.Policies[p_name][panel][i][k]['args'])
except:
self.assertEqual(p_res[p_name][i][k],
self.configModule.Policies[p_name][panel][i][k])
else:
self.assertEqual(p_res[p_name], self.configModule.Policies[p_name][panel])
def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None):
""" PDP MAIN FUNCTION
decides policies that have to be applied, based on
__granularity,
__name,
__status,
__formerStatus
__reason
If more than one policy is evaluated, results are combined.
Logic for combination: a conservative approach is followed
(i.e. if a site should be banned for at least one policy, that's what is returned)
returns:
{ 'PolicyType': a policyType (in a string),
'Action': True|False,
'Status': 'Active'|'Probing'|'Banned',
'Reason': a reason
'EndDate: datetime.datetime (in a string)}
"""
self.args = argsIn
self.policy = policyIn
self.knownInfo = knownInfo
self.ig = InfoGetter(self.VOExtension)
EVAL = self.ig.getInfoToApply(('policy', 'policyType'),
granularity=self.__granularity,
status=self.__status,
formerStatus=self.__formerStatus,
siteType=self.__siteType,
serviceType=self.__serviceType,
resourceType=self.__resourceType,
useNewRes=self.useNewRes)
for policyGroup in EVAL:
policyType = policyGroup['PolicyType']
if self.policy is not None:
# Only the policy provided will be evaluated
# FIXME: Check that the policies are valid.
singlePolicyResults = self.policy.evaluate()
else:
if policyGroup['Policies'] is None:
return {
'SinglePolicyResults': [],
'PolicyCombinedResult': {
'PolicyType': policyType,
'Action': False,
'Reason': 'No policy results'
}
}
else:
singlePolicyResults = self._invocation(
self.VOExtension, self.__granularity, self.__name,
self.__status, self.policy, self.args,
policyGroup['Policies'])
policyCombinedResults = self._policyCombination(
singlePolicyResults)
assert (type(policyCombinedResults) == dict)
if not policyCombinedResults:
return {
'SinglePolicyResults': singlePolicyResults,
'PolicyCombinedResult': {}
}
#
# policy results communication
#
newstatus = policyCombinedResults['Status']
if newstatus != self.__status: # Policies satisfy
reason = policyCombinedResults['Reason']
newPolicyType = self.ig.getNewPolicyType(
self.__granularity, newstatus)
for npt in newPolicyType:
if npt not in policyType:
policyType.append(npt)
decision = {
'PolicyType': policyType,
'Action': True,
'Status': newstatus,
'Reason': reason
}
if policyCombinedResults.has_key('EndDate'):
decision['EndDate'] = policyCombinedResults['EndDate']
else: # Policies does not satisfy
reason = policyCombinedResults['Reason']
decision = {
'PolicyType': policyType,
'Action': False,
'Reason': reason
}
if policyCombinedResults.has_key('EndDate'):
decision['EndDate'] = policyCombinedResults['EndDate']
res = {
'SinglePolicyResults': singlePolicyResults,
'PolicyCombinedResult': decision
}
assert (type(res) == dict)
return res
