def __init__(self, clients=None): """ Constructor. examples: >>> pdp = PDP( None ) >>> pdp1 = PDP( {} ) >>> pdp2 = PDP( { 'Client1' : Client1Object } ) :Parameters: **clients** - [ None, `dict` ] dictionary with Clients to be used in the Commands. If None, the Commands will create their own clients. """ # decision parameters used to match policies and actions self.decisionParams = None # Helpers to discover policies and RSS metadata in CS self.iGetter = InfoGetter() self.pCaller = PolicyCaller(clients) # RSS State Machine, used to calculate most penalizing state while merging them self.rssMachine = RSSMachine('Unknown') self.log = gLogger.getSubLogger('PDP')
def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport( "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {}
def __init__(self, clients): ''' Constructor. Defines members that will be used later on. ''' self.pCaller = PolicyCaller(clients=clients) self.iGetter = InfoGetter() self.decissionParams = {} self.rssMachine = RSSMachine('Unknown')
def initializeResourceManagementHandler(serviceInfo): global rsDB rsDB = ResourceStatusDB() global rmDB rmDB = ResourceManagementDB() cc = CommandCaller() global VOExtension VOExtension = getExt() ig = InfoGetter(VOExtension) WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") global publisher publisher = Publisher(VOExtension, rsDBIn=rsDB, commandCallerIn=cc, infoGetterIn=ig, WMSAdminIn=WMSAdmin) # sync_O = Synchronizer(rsDB) # gConfig.addListenerToNewVersionEvent( sync_O.sync ) return S_OK()
def __init__( self, clients = None ): """ Constructor. examples: >>> pdp = PDP( None ) >>> pdp1 = PDP( {} ) >>> pdp2 = PDP( { 'Client1' : Client1Object } ) :Parameters: **clients** - [ None, `dict` ] dictionary with Clients to be used in the Commands. If None, the Commands will create their own clients. """ # decision parameters used to match policies and actions self.decisionParams = None # Helpers to discover policies and RSS metadata in CS self.iGetter = InfoGetter() self.pCaller = PolicyCaller( clients ) # RSS State Machine, used to calculate most penalizing state while merging them self.rssMachine = RSSMachine( 'Unknown' ) self.log = gLogger.getSubLogger( 'PDP' )
def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None self.infoGetter = InfoGetter() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache('StorageElement', cacheLifeTime, self.__updateSECache)
def __init__(self, clients): """ Constructor. Defines members that will be used later on. """ self.pCaller = PolicyCaller(clients=clients) self.iGetter = InfoGetter() self.decissionParams = {} self.rssMachine = RSSMachine("Unknown")
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None, WMSAdminIn = None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool( 2, 5 ) self.lockObj = threading.RLock() self.infoForPanel_res = {}
def __init__(self, **clients): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller(cc, **clients) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None
def __init__( self ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None self.infoGetter = InfoGetter() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( 'StorageElement', cacheLifeTime, self.__updateSECache )
def setUp(self): sys.modules["DIRAC"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.ResourceStatusSystem.Utilities.CS"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Core.Utilities.SiteCEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Core.Utilities.SiteSEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Core.Utilities.SitesDIRACGOCDBmapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger sys.modules[ "DIRAC.Interfaces.API.DiracAdmin"] = DIRAC.ResourceStatusSystem.test.fake_Admin sys.modules[ "DIRAC.FrameworkSystem.Client.NotificationClient"] = DIRAC.ResourceStatusSystem.test.fake_NotificationClient from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter from DIRAC.ResourceStatusSystem.PolicySystem.PolicyBase import PolicyBase from DIRAC.ResourceStatusSystem.PolicySystem.PolicyInvoker import PolicyInvoker from DIRAC import gConfig self.VO = gConfig.getValue("DIRAC/Extensions") if 'LHCb' in self.VO: self.VO = 'LHCb' self.mock_command = Mock() self.mock_policy = Mock() self.mock_p = Mock() self.mock_args = Mock() self.pb = PolicyBase() self.pi = PolicyInvoker() self.mock_pdp = Mock() self.mock_rsDB = Mock() self.mock_rmDB = Mock() self.mock_nc = Mock() self.mock_da = Mock() self.mock_da.getBannedSites.return_value = { 'OK': True, 'Value': ['LCG.APC.fr', 'LCG.Bari.it', 'LCG.Catania.it'] } self.mock_da.addSiteInMask.return_value = {'OK': True, 'Value': ''} self.mock_da.banSiteFromMask.return_value = {'OK': True, 'Value': ''} self.mock_da.sendMail.return_value = {'OK': True, 'Value': ''} self.mock_csAPI = Mock() self.mock_csAPI.setOption.return_value = {'OK': True, 'Value': ''} self.mock_csAPI.commit.return_value = {'OK': True, 'Value': ''} self.ig = InfoGetter(self.VO)
def __init__( self, **clients ): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller( cc, **clients ) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None
def testGetInfoToApply(self): ig = InfoGetter('LHCb') for g in ValidRes: for s in ValidStatus: for site_t in ValidSiteType: for service_t in ValidServiceType: if g in ('Site', 'Sites'): panel = 'Site_Panel' if g in ('Service', 'Services'): if service_t == 'Storage': panel = 'Service_Storage_Panel' if service_t == 'Computing': panel = 'Service_Computing_Panel' if service_t == 'VO-BOX': panel = 'Service_VO-BOX_Panel' if service_t == 'VOMS': panel = 'Service_VOMS_Panel' if g in ('Resource', 'Resources'): panel = 'Resource_Panel' if g in ('StorageElementRead', 'StorageElementsRead'): panel = 'SE_Panel' if g in ('StorageElementWrite', 'StorageElementsWrite'): panel = 'SE_Panel' for resource_t in ValidResourceType: res = ig.getInfoToApply(('policyType', ), g, s, None, site_t, service_t, resource_t) for p_res in res[0]['PolicyType']: self.assert_(p_res in ['Resource_PolType', 'Alarm_PolType', 'Alarm_PolType_SE']) for useNewRes in (False, True): res = ig.getInfoToApply(('policy', ), g, s, None, site_t, service_t, resource_t, useNewRes) pModuleList = [None] for k in self.configModule.Policies.keys(): try: if self.configModule.Policies[k]['module'] not in pModuleList: pModuleList.append(self.configModule.Policies[k]['module']) except KeyError: pass for p_res in res[0]['Policies']: self.assert_(p_res['Name'] in self.configModule.Policies.keys()) self.assert_(p_res['Module'] in pModuleList) if useNewRes is False: self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandIn']) self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['args']) else: try: self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandInNewRes']) except KeyError: self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandIn']) try: self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['argsNewRes']) except KeyError: self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['args']) res = ig.getInfoToApply(('panel_info', ), g, s, None, site_t, service_t, resource_t, useNewRes) for p_res in res[0]['Info']: # if 'JobsEfficiencySimple' in p_res.keys(): # print useNewRes, p_res for p_name in p_res.keys(): self.assert_(p_name in self.configModule.Policies.keys()) if isinstance(p_res[p_name], list): for i in range(len(p_res[p_name])): for k in p_res[p_name][i].keys(): if useNewRes: try: self.assertEqual(p_res[p_name][i][k]['CommandIn'], self.configModule.Policies[p_name][panel][i][k]['CommandInNewRes']) except KeyError: self.assertEqual(p_res[p_name][i][k]['CommandIn'], self.configModule.Policies[p_name][panel][i][k]['CommandIn']) except TypeError: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) try: self.assertEqual(p_res[p_name][i][k]['args'], self.configModule.Policies[p_name][panel][i][k]['argsNewRes']) except KeyError: self.assertEqual(p_res[p_name][i][k]['args'], self.configModule.Policies[p_name][panel][i][k]['args']) except TypeError: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) else: try: self.assertEqual(p_res[p_name][i][k]['CommandIn'], self.configModule.Policies[p_name][panel][i][k]['CommandIn']) except: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) try: self.assertEqual(p_res[p_name][i][k]['args'], self.configModule.Policies[p_name][panel][i][k]['args']) except: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) else: self.assertEqual(p_res[p_name], self.configModule.Policies[p_name][panel])
def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ self.args = argsIn self.policy = policyIn self.knownInfo = knownInfo self.ig = InfoGetter(self.VOExtension) EVAL = self.ig.getInfoToApply(('policy', 'policyType'), granularity=self.__granularity, status=self.__status, formerStatus=self.__formerStatus, siteType=self.__siteType, serviceType=self.__serviceType, resourceType=self.__resourceType, useNewRes=self.useNewRes) for policyGroup in EVAL: policyType = policyGroup['PolicyType'] if self.policy is not None: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = self.policy.evaluate() else: if policyGroup['Policies'] is None: return { 'SinglePolicyResults': [], 'PolicyCombinedResult': { 'PolicyType': policyType, 'Action': False, 'Reason': 'No policy results' } } else: singlePolicyResults = self._invocation( self.VOExtension, self.__granularity, self.__name, self.__status, self.policy, self.args, policyGroup['Policies']) policyCombinedResults = self._policyCombination( singlePolicyResults) assert (type(policyCombinedResults) == dict) if not policyCombinedResults: return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': {} } # # policy results communication # newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy reason = policyCombinedResults['Reason'] newPolicyType = self.ig.getNewPolicyType( self.__granularity, newstatus) for npt in newPolicyType: if npt not in policyType: policyType.append(npt) decision = { 'PolicyType': policyType, 'Action': True, 'Status': newstatus, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] else: # Policies does not satisfy reason = policyCombinedResults['Reason'] decision = { 'PolicyType': policyType, 'Action': False, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] res = { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': decision } assert (type(res) == dict) return res
class PDP: """ PDP = Policy Decision Point. Used to invoke policies and to take decision based on the polict results combination. """ ############################################################################# def __init__(self, VOExtension, granularity=None, name=None, status=None, formerStatus=None, reason=None, siteType=None, serviceType=None, resourceType=None, useNewRes=False): """ PDP (Policy Decision Point) initialization :params: :attr:`VOExtension`: string - VO extension (e.g. 'LHCb') :attr:`granularity`: string - a ValidRes :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.VOExtension = VOExtension self.__granularity = assignOrRaise(granularity, ValidRes, InvalidRes, self, self.__init__) self.__name = name self.__status = assignOrRaise(status, ValidStatus, InvalidStatus, self, self.__init__) self.__formerStatus = assignOrRaise(formerStatus, ValidStatus, InvalidStatus, self, self.__init__) self.__reason = reason self.__siteType = assignOrRaise(siteType, ValidSiteType, InvalidSiteType, self, self.__init__) self.__serviceType = assignOrRaise(serviceType, ValidServiceType, InvalidServiceType, self, self.__init__) self.__resourceType = assignOrRaise(resourceType, ValidResourceType, InvalidResourceType, self, self.__init__) cc = CommandCaller() self.pc = PolicyCaller(cc) self.useNewRes = useNewRes self.args = None self.policy = None self.knownInfo = None self.ig = None ############################################################################# def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ self.args = argsIn self.policy = policyIn self.knownInfo = knownInfo self.ig = InfoGetter(self.VOExtension) EVAL = self.ig.getInfoToApply(('policy', 'policyType'), granularity=self.__granularity, status=self.__status, formerStatus=self.__formerStatus, siteType=self.__siteType, serviceType=self.__serviceType, resourceType=self.__resourceType, useNewRes=self.useNewRes) for policyGroup in EVAL: policyType = policyGroup['PolicyType'] if self.policy is not None: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = self.policy.evaluate() else: if policyGroup['Policies'] is None: return { 'SinglePolicyResults': [], 'PolicyCombinedResult': { 'PolicyType': policyType, 'Action': False, 'Reason': 'No policy results' } } else: singlePolicyResults = self._invocation( self.VOExtension, self.__granularity, self.__name, self.__status, self.policy, self.args, policyGroup['Policies']) policyCombinedResults = self._policyCombination( singlePolicyResults) assert (type(policyCombinedResults) == dict) if not policyCombinedResults: return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': {} } # # policy results communication # newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy reason = policyCombinedResults['Reason'] newPolicyType = self.ig.getNewPolicyType( self.__granularity, newstatus) for npt in newPolicyType: if npt not in policyType: policyType.append(npt) decision = { 'PolicyType': policyType, 'Action': True, 'Status': newstatus, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] else: # Policies does not satisfy reason = policyCombinedResults['Reason'] decision = { 'PolicyType': policyType, 'Action': False, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] res = { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': decision } assert (type(res) == dict) return res ############################################################################# def _invocation(self, VOExtension, granularity, name, status, policy, args, policies): """ One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. """ policyResults = [] for p in policies: pName = p['Name'] pModule = p['Module'] extraArgs = p['args'] commandIn = p['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity=granularity, name=name, status=status, policy=policy, args=args, pName=pName, pModule=pModule, extraArgs=extraArgs, commandIn=commandIn) # If res is empty, return immediately if not res: return policyResults if not res.has_key('Status'): print("\n\n Policy result " + str(res) + " does not return 'Status'\n\n") raise TypeError # Else if res['Status'] == 'Unknown': res = self.__useOldPolicyRes(name=name, policyName=pName) if res['Status'] == 'NeedConfirmation': pName = p['ConfirmationPolicy'] triggeredPolicy = self.ig.C_Policies[pName] pModule = triggeredPolicy['module'] extraArgs = triggeredPolicy['args'] commandIn = triggeredPolicy['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity=granularity, name=name, status=status, policy=policy, args=args, pName=pName, pModule=pModule, extraArgs=extraArgs, commandIn=commandIn) if res['Status'] not in ('Error', 'Unknown', 'NeedConfirmation'): policyResults.append(res) return policyResults ############################################################################# def _policyCombination(self, pol_results): """ INPUT: list type OUTPUT: dict type * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result """ if pol_results == []: return {} pol_results.sort(key=Status.value_of_policy) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions _prio, access_list, gofun = Status.statesInfo[self.__status] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for p in pol_results: if Status.value_of_policy(p) in access_list: newStatus = Status.value_of_policy(p) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun(access_list) return { 'Status': Status.status_of_value(newStatus), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = Status.value_of_policy(pol_results[0]) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = Status.value_of_policy(pol_results[0]) # At this point, a new status has been chosen. newStatus is an # integer. worstResults = [ p for p in pol_results if Status.value_of_policy(p) == newStatus ] # Concatenate reasons def getReason(p): try: res = p['Reason'] except KeyError: res = '' return res worstResultsReasons = [getReason(p) for p in worstResults] def catRes(x, y): if x and y: return x + ' |###| ' + y elif x or y: if x: return x else: return y else: return '' concatenatedRes = reduce(catRes, worstResultsReasons, '') # Handle EndDate endDatePolicies = [p for p in worstResults if p.has_key('EndDate')] # Building and returning result res = {} res['Status'] = Status.status_of_value(newStatus) if concatenatedRes != '': res['Reason'] = concatenatedRes if endDatePolicies != []: res['EndDate'] = endDatePolicies[0]['EndDate'] return res ############################################################################# def __useOldPolicyRes(self, name, policyName): """ Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} """ from DIRAC.Core.DISET.RPCClient import RPCClient rsS = RPCClient("ResourceStatus/ResourceManagement") res = rsS.getPolicyRes(name, policyName, True) if not res['OK']: raise RSSException, where( self, self.__useOldPolicyRes) + ' Could not get a policy result' res = res['Value'] if res == []: return {'Status': 'Unknown'} oldStatus = res[0] oldReason = res[1] lastCheckTime = res[2] if (lastCheckTime + datetime.timedelta(hours=2)) < datetime.datetime.utcnow(): return {'Status': 'Unknown'} result = {} result['Status'] = oldStatus result['Reason'] = oldReason result['OLD'] = True result['PolicyName'] = policyName return result
class PDP: """ PDP ( Policy Decision Point ) """ def __init__(self, clients=None): """ Constructor. examples: >>> pdp = PDP( None ) >>> pdp1 = PDP( {} ) >>> pdp2 = PDP( { 'Client1' : Client1Object } ) :Parameters: **clients** - [ None, `dict` ] dictionary with Clients to be used in the Commands. If None, the Commands will create their own clients. """ # decission parameters used to match policies and actions self.decisionParams = {} # Helpers to discover policies and RSS metadata in CS self.iGetter = InfoGetter() self.pCaller = PolicyCaller(clients) # RSS State Machine, used to calculate most penalizing state while merging them self.rssMachine = RSSMachine('Unknown') def setup(self, decisionParams=None): """ method that sanitizes the decisionParams and ensures that at least it has the keys in `standardParamsDict`. This will be relevant while doing the matching with the RSS Policies configuration in the CS. There is one key-value pair, `active` which is added on this method. This allows policies to be de-activated from the CS, changing their active matchParam to something else than `Active`. examples: >>> pdp.setup( None ) >>> self.decisionParams { 'element' : None, 'name' : None, ... } >>> pdp.setup( { 'element' : 'AnElement' } ) >>> self.decisionParams { 'element' : 'AnElement', 'name' : None, ... } >>> pdp.setup( { 'NonStandardKey' : 'Something' } ) >>> self.decisionParams { 'NonStandardKey' : 'Something', 'element' : None,... } :Parameters: **decisionParams** - [ None, `dict` ] dictionary with the parameters to be matched with the RSS Policies configuration in the CS. """ standardParamsDict = { 'element': None, 'name': None, 'elementType': None, 'statusType': None, 'status': None, 'reason': None, 'tokenOwner': None, # Last parameter allows policies to be de-activated 'active': 'Active' } if decisionParams is not None: for key in standardParamsDict: try: standardParamsDict[key] = decisionParams[key] except KeyError: pass self.decisionParams = standardParamsDict def takeDecision(self): """ main PDP method which does all the work. If firstly finds all the policies defined in the CS that match <self.decisionParams> and runs them. Once it has all the singlePolicyResults, it combines them. Next step is action discovery: using a similar approach to the one used to discover the policies, but also taking into account the single policy results and their combined result, finds the actions to be triggered and returns. examples: >>> pdp.takeDecision()[ 'Value' ].keys() [ 'singlePolicyResults', 'policyCombinedResult', 'decissionParams' ] >>> pdp.takeDecision()[ 'Value' ][ 'singlePolicyResults' ] [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : { 'name' : 'AlwaysActiveForResource', 'type' : 'AlwaysActive', 'module' : 'AlwaysActivePolicy', 'description' : 'This is the AlwaysActive policy' 'command' : None, 'args' : {} } }, ... ] >>> pdp.takeDecision()[ 'Value' ][ 'policyCombinedResult' ] { 'Status' : 'Active', 'Reason' : 'blah ###', 'PolicyAction' : [ ( 'policyActionName1', 'policyActionType1' ), ... ] } :return: S_OK( { 'singlePolicyResults' : `list`, 'policyCombinedResult' : `dict`, 'decissionParams' : `dict` } ) / S_ERROR """ # Policies.................................................................. # Get policies that match self.decisionParams policiesThatApply = self.iGetter.getPoliciesThatApply( self.decisionParams) if not policiesThatApply['OK']: return policiesThatApply policiesThatApply = policiesThatApply['Value'] # Evaluate policies singlePolicyResults = self._runPolicies(policiesThatApply) if not singlePolicyResults['OK']: return singlePolicyResults singlePolicyResults = singlePolicyResults['Value'] # Combine policies and get most penalizing status ( see RSSMachine ) policyCombinedResults = self._combineSinglePolicyResults( singlePolicyResults) if not policyCombinedResults['OK']: return policyCombinedResults policyCombinedResults = policyCombinedResults['Value'] # Actions................................................................... policyActionsThatApply = self.iGetter.getPolicyActionsThatApply( self.decisionParams, singlePolicyResults, policyCombinedResults) if not policyActionsThatApply['OK']: return policyActionsThatApply policyActionsThatApply = policyActionsThatApply['Value'] policyCombinedResults['PolicyAction'] = policyActionsThatApply return S_OK({ 'singlePolicyResults': singlePolicyResults, 'policyCombinedResult': policyCombinedResults, 'decissionParams': self.decisionParams }) def _runPolicies(self, policies): """ Given a list of policy dictionaries, loads them making use of the PolicyCaller and evaluates them. This method requires to have run setup previously. examples: >>> pdp._runPolicies([])[ 'Value' ] [] >>> policyDict = { 'name' : 'AlwaysActiveResource', 'type' : 'AlwaysActive', 'args' : None, 'description' : 'This is the AlwaysActive policy', 'module' : 'AlwaysActivePolicy', 'command' : None } >>> pdp._runPolicies([ policyDict, ... ] )[ 'Value' ] [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict }, ... ] :Parameters: **policies** - `list( dict )` list of dictionaries containing the policies selected to be run. Check the examples to get an idea of how the policy dictionaries look like. :return: S_OK() / S_ERROR """ policyInvocationResults = [] # Gets all valid status for RSS to avoid misconfigured policies returning statuses # that RSS does not understand. validStatus = self.rssMachine.getStates() for policyDict in policies: # Load and evaluate policy described in <policyDict> for element described # in <self.decisionParams> policyInvocationResult = self.pCaller.policyInvocation( self.decisionParams, policyDict) if not policyInvocationResult['OK']: # We should never enter this line ! Just in case there are policies # missconfigured ! _msg = 'runPolicies no OK: %s' % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) policyInvocationResult = policyInvocationResult['Value'] # Sanity Checks ( they should never happen ! ) if not 'Status' in policyInvocationResult: _msg = 'runPolicies (no Status): %s' % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) if not policyInvocationResult['Status'] in validStatus: _msg = 'runPolicies ( not valid status ) %s' % policyInvocationResult[ 'Status'] gLogger.error(_msg) return S_ERROR(_msg) if not 'Reason' in policyInvocationResult: _msg = 'runPolicies (no Reason): %s' % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) policyInvocationResults.append(policyInvocationResult) return S_OK(policyInvocationResults) def _combineSinglePolicyResults(self, singlePolicyRes): """ method that merges all the policies results into a combined one, which will be the most penalizing status and the reasons of the single policy results that returned the same penalizing status. All the rest, are ignored. If there are no single policy results, it is returned `Unknown` state. While combining policies, the ones containing the option `doNotCombine` are ignored. examples: >>> pdp._combineSingePolicyResults( [] )[ 'Value' ] { 'Status' : 'Unknown', 'Reason' : 'No policy ..' } >>> pdp._combineSingePolicyResults( [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict } ] ) { 'Status' : 'Active', 'Reason' : 'blah' } >>> pdp._combineSingePolicyResults( [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict }, { 'Status' : 'Banned', 'Reason' : 'blah 2', 'Policy' : policyDict2 } ] ) { 'Status' : 'Banned', 'Reason' : 'blah 2' } >>> pdp._combineSingePolicyResults( [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict }, { 'Status' : 'Active', 'Reason' : 'blah 2', 'Policy' : policyDict2 } ] ) { 'Status' : 'Banned', 'Reason' : 'blah ### blah 2' } :Parameters: **singlePolicyRes** - `list( dict )` list with every single policy result to be combined ( see _runPolicy for more details ) :return: S_OK( dict( Status, Reason ) | S_ERROR """ # Dictionary to be returned policyCombined = {'Status': 'Unknown', 'Reason': ''} # If there are no policyResults, we return Unknown if not singlePolicyRes: policyCombined['Status'] = 'Unknown' policyCombined[ 'Reason'] = 'No policy applies to %(element)s, %(name)s, %(elementType)s' % self.decisionParams return S_OK(policyCombined) # We set the rssMachine on the current state ( ensures it is a valid one ) # FIXME: probably this check can be done at takeDecission machineStatus = self.rssMachine.setState(self.decisionParams['status']) if not machineStatus['OK']: return machineStatus # Discard all single policy results which belogs to policies that have set # the option `doNotCombine` in the CS policiesToCombine = self._findPoliciesToCombine(singlePolicyRes) # Sort policy results using ther statuses by most restrictive ( lower level first ) self.rssMachine.orderPolicyResults(policiesToCombine) # As they have been sorted by most restrictive status, the first one is going # to be our candidate new state. Let's ask the RSSMachine if it allows us to # make such transition. candidateState = policiesToCombine[0]['Status'] nextState = self.rssMachine.getNextState(candidateState) if not nextState['OK']: return nextState nextState = nextState['Value'] # If the RssMachine does not accept the candidate, return forcing message if candidateState != nextState: policyCombined['Status'] = nextState policyCombined['Reason'] = 'RssMachine forced status %s to %s' % ( candidateState, nextState) return S_OK(policyCombined) # If the RssMachine accepts the candidate, just concatenate the reasons for policyRes in policiesToCombine: if policyRes['Status'] == nextState: policyCombined['Reason'] += '%s ###' % policyRes['Reason'] policyCombined['Status'] = nextState return S_OK(policyCombined) def _findPoliciesToCombine(self, singlePolicyRes): """ method that iterates over the single policy results and checks the CS configuration of the policies looking for the option 'doNotCombine'. If it is present, that single policy result is discarded. :Parameters: **singlePolicyRes** - `list( dict )` list with every single policy result to be combined ( see _runPolicy for more details ) :return: `list( dict )` """ # Get policies configuration from the CS. We want to exclude the policies that # have set the option `doNotCombine` from this process. policiesConfiguration = RssConfiguration.getPolicies() if not policiesConfiguration['OK']: return policiesConfiguration policiesConfiguration = policiesConfiguration['Value'] # Function that let's us know if we should combine the result of a single policy # or not. def combinePolicy(policyResult): # Extract policy name from the dictionary returned by PolicyCaller policyName = policyResult['Policy']['name'] try: # If doNotCombineResult is defined, the policy is not taken into account # to create the combined result. However, the single policy result remains _ = policiesConfiguration[policyName]['doNotCombineResult'] return False except KeyError: return True # Make a list of policies of which we want to merge their results return [ policyResult for policyResult in singlePolicyRes if combinePolicy(policyResult) ] #............................................................................... #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__( self, clients ): ''' Constructor. Defines members that will be used later on. ''' self.pCaller = PolicyCaller( clients = clients ) self.iGetter = InfoGetter() self.decissionParams = {} self.rssMachine = RSSMachine( 'Unknown' ) def setup( self, decissionParams = None ): standardParamsDict = { 'element' : None, 'name' : None, 'elementType' : None, 'statusType' : None, 'status' : None, 'reason' : None, 'tokenOwner' : None, # Last parameter allows policies to be deactivated 'active' : 'Active' } if decissionParams is not None: standardParamsDict.update( decissionParams ) self.decissionParams = standardParamsDict ################################################################################ def takeDecision( self ):#, policyIn = None, argsIn = None, knownInfo = None ): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason #'EndDate: datetime.datetime (in a string)} """ policiesThatApply = self.iGetter.getPoliciesThatApply( self.decissionParams ) if not policiesThatApply[ 'OK' ]: return policiesThatApply policiesThatApply = policiesThatApply[ 'Value' ] singlePolicyResults = self._runPolicies( policiesThatApply ) if not singlePolicyResults[ 'OK' ]: return singlePolicyResults singlePolicyResults = singlePolicyResults[ 'Value' ] policyCombinedResults = self._combineSinglePolicyResults( singlePolicyResults ) if not policyCombinedResults[ 'OK' ]: return policyCombinedResults policyCombinedResults = policyCombinedResults[ 'Value' ] #FIXME: should also pass the result of the combination to the InfoGetter ? policyActionsThatApply = self.iGetter.getPolicyActionsThatApply( self.decissionParams ) if not policyActionsThatApply[ 'OK' ]: return policyActionsThatApply policyActionsThatApply = policyActionsThatApply[ 'Value' ] policyCombinedResults[ 'PolicyAction' ] = policyActionsThatApply return S_OK( { 'singlePolicyResults' : singlePolicyResults, 'policyCombinedResult' : policyCombinedResults, 'decissionParams' : self.decissionParams } ) ################################################################################ def _runPolicies( self, policies, decissionParams = None ): if decissionParams is None: decissionParams = self.decissionParams validStatus = RssConfiguration.getValidStatus() if not validStatus[ 'OK' ]: return validStatus validStatus = validStatus[ 'Value' ] policyInvocationResults = [] for policyDict in policies: policyInvocationResult = self.pCaller.policyInvocation( decissionParams, policyDict ) if not policyInvocationResult[ 'OK' ]: # We should never enter this line ! Just in case there are policies # missconfigured ! _msg = 'runPolicies no OK: %s' % policyInvocationResult gLogger.error( _msg ) return S_ERROR( _msg ) policyInvocationResult = policyInvocationResult[ 'Value' ] if not 'Status' in policyInvocationResult: _msg = 'runPolicies (no Status): %s' % policyInvocationResult gLogger.error( _msg ) return S_ERROR( _msg ) if not policyInvocationResult[ 'Status' ] in validStatus: _msg = 'runPolicies ( not valid status ) %s' % policyInvocationResult[ 'Status' ] gLogger.error( _msg ) return S_ERROR( _msg ) if not 'Reason' in policyInvocationResult: _msg = 'runPolicies (no Reason): %s' % policyInvocationResult gLogger.error( _msg ) return S_ERROR( _msg ) policyInvocationResults.append( policyInvocationResult ) return S_OK( policyInvocationResults ) ################################################################################ def _combineSinglePolicyResults( self, singlePolicyRes ): ''' singlePolicyRes = [ { 'State' : X, 'Reason' : Y, ... }, ... ] If there are no policyResults, returns Unknown as there are no policies to apply. Order elements in list by state, being the lowest the most restrictive one in the hierarchy. ''' # Dictionary to be returned policyCombined = { 'Status' : None, 'Reason' : '' } # If there are no policyResults, we return Unknown if not singlePolicyRes: _msgTuple = ( self.decissionParams[ 'element' ], self.decissionParams[ 'name' ], self.decissionParams[ 'elementType' ] ) policyCombined[ 'Status' ] = 'Unknown' policyCombined[ 'Reason' ] = 'No policy applies to %s, %s, %s' % _msgTuple return S_OK( policyCombined ) # We set the rssMachine on the current state machineStatus = self.rssMachine.setState( self.decissionParams[ 'status' ] ) if not machineStatus[ 'OK' ]: return machineStatus # Order statuses by most restrictive ( lower level first ) self.rssMachine.orderPolicyResults( singlePolicyRes ) #policyResults = self.rssMachine.orderPolicyResults( singlePolicyRes ) # Get according to the RssMachine the next state, given a candidate candidateState = singlePolicyRes[ 0 ][ 'Status' ] nextState = self.rssMachine.getNextState( candidateState ) if not nextState[ 'OK' ]: return nextState nextState = nextState[ 'Value' ] # If the RssMachine does not accept the candidate, return forcing message if candidateState != nextState: policyCombined[ 'Status' ] = nextState policyCombined[ 'Reason' ] = 'RssMachine forced status %s to %s' % ( candidateState, nextState ) return S_OK( policyCombined ) # If the RssMachine accepts the candidate, just concatenate the reasons for policyRes in singlePolicyRes: if policyRes[ 'Status' ] == nextState: policyCombined[ 'Reason' ] += '%s ###' % policyRes[ 'Reason' ] policyCombined[ 'Status' ] = nextState return S_OK( policyCombined ) ################################################################################ # def __useOldPolicyRes( self, name, policyName ): # ''' # Use the RSS Service to get an old policy result. # If such result is older than 2 hours, it returns {'Status':'Unknown'} # ''' # res = self.clients[ 'ResourceManagementClient' ].getPolicyResult( name = name, policyName = policyName ) # # if not res[ 'OK' ]: # return { 'Status' : 'Unknown' } # # res = res[ 'Value' ] # # if res == []: # return { 'Status' : 'Unknown' } # # res = res[ 0 ] # # oldStatus = res[ 5 ] # oldReason = res[ 6 ] # lastCheckTime = res[ 8 ] # # if ( lastCheckTime + datetime.timedelta(hours = 2) ) < datetime.datetime.utcnow(): # return { 'Status' : 'Unknown' } # # result = {} # # result[ 'Status' ] = oldStatus # result[ 'Reason' ] = oldReason # result[ 'OLD' ] = True # result[ 'PolicyName' ] = policyName # # return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__(self, clients): """ Constructor. Defines members that will be used later on. """ self.pCaller = PolicyCaller(clients=clients) self.iGetter = InfoGetter() self.decissionParams = {} self.rssMachine = RSSMachine("Unknown") def setup(self, decissionParams=None): standardParamsDict = { "element": None, "name": None, "elementType": None, "statusType": None, "status": None, "reason": None, "tokenOwner": None, # Last parameter allows policies to be deactivated "active": "Active", } if decissionParams is not None: standardParamsDict.update(decissionParams) self.decissionParams = standardParamsDict ################################################################################ def takeDecision(self): # , policyIn = None, argsIn = None, knownInfo = None ): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason #'EndDate: datetime.datetime (in a string)} """ policiesThatApply = self.iGetter.getPoliciesThatApply(self.decissionParams) if not policiesThatApply["OK"]: return policiesThatApply policiesThatApply = policiesThatApply["Value"] singlePolicyResults = self._runPolicies(policiesThatApply) if not singlePolicyResults["OK"]: return singlePolicyResults singlePolicyResults = singlePolicyResults["Value"] policyCombinedResults = self._combineSinglePolicyResults(singlePolicyResults) if not policyCombinedResults["OK"]: return policyCombinedResults policyCombinedResults = policyCombinedResults["Value"] policyActionsThatApply = self.iGetter.getPolicyActionsThatApply( self.decissionParams, singlePolicyResults, policyCombinedResults ) if not policyActionsThatApply["OK"]: return policyActionsThatApply policyActionsThatApply = policyActionsThatApply["Value"] policyCombinedResults["PolicyAction"] = policyActionsThatApply return S_OK( { "singlePolicyResults": singlePolicyResults, "policyCombinedResult": policyCombinedResults, "decissionParams": self.decissionParams, } ) ################################################################################ def _runPolicies(self, policies, decissionParams=None): if decissionParams is None: decissionParams = self.decissionParams validStatus = RssConfiguration.getValidStatus() if not validStatus["OK"]: return validStatus validStatus = validStatus["Value"] policyInvocationResults = [] for policyDict in policies: policyInvocationResult = self.pCaller.policyInvocation(decissionParams, policyDict) if not policyInvocationResult["OK"]: # We should never enter this line ! Just in case there are policies # missconfigured ! _msg = "runPolicies no OK: %s" % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) policyInvocationResult = policyInvocationResult["Value"] if not "Status" in policyInvocationResult: _msg = "runPolicies (no Status): %s" % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) if not policyInvocationResult["Status"] in validStatus: _msg = "runPolicies ( not valid status ) %s" % policyInvocationResult["Status"] gLogger.error(_msg) return S_ERROR(_msg) if not "Reason" in policyInvocationResult: _msg = "runPolicies (no Reason): %s" % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) policyInvocationResults.append(policyInvocationResult) return S_OK(policyInvocationResults) ################################################################################ def _combineSinglePolicyResults(self, singlePolicyRes): """ singlePolicyRes = [ { 'State' : X, 'Reason' : Y, ... }, ... ] If there are no policyResults, returns Unknown as there are no policies to apply. Order elements in list by state, being the lowest the most restrictive one in the hierarchy. """ # Dictionary to be returned policyCombined = {"Status": None, "Reason": ""} # If there are no policyResults, we return Unknown if not singlePolicyRes: _msgTuple = ( self.decissionParams["element"], self.decissionParams["name"], self.decissionParams["elementType"], ) policyCombined["Status"] = "Unknown" policyCombined["Reason"] = "No policy applies to %s, %s, %s" % _msgTuple return S_OK(policyCombined) # We set the rssMachine on the current state machineStatus = self.rssMachine.setState(self.decissionParams["status"]) if not machineStatus["OK"]: return machineStatus # Order statuses by most restrictive ( lower level first ) self.rssMachine.orderPolicyResults(singlePolicyRes) # policyResults = self.rssMachine.orderPolicyResults( singlePolicyRes ) # Get according to the RssMachine the next state, given a candidate candidateState = singlePolicyRes[0]["Status"] nextState = self.rssMachine.getNextState(candidateState) if not nextState["OK"]: return nextState nextState = nextState["Value"] # If the RssMachine does not accept the candidate, return forcing message if candidateState != nextState: policyCombined["Status"] = nextState policyCombined["Reason"] = "RssMachine forced status %s to %s" % (candidateState, nextState) return S_OK(policyCombined) # If the RssMachine accepts the candidate, just concatenate the reasons for policyRes in singlePolicyRes: if policyRes["Status"] == nextState: policyCombined["Reason"] += "%s ###" % policyRes["Reason"] policyCombined["Status"] = nextState return S_OK(policyCombined)
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__(self, **clients): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller(cc, **clients) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None def setup(self, granularity=None, name=None, statusType=None, status=None, formerStatus=None, reason=None, siteType=None, serviceType=None, resourceType=None, useNewRes=False): """ PDP (Policy Decision Point) initialization :params: :attr:`granularity`: string - a ValidElement :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.__granularity = granularity self.__name = name self.__statusType = statusType self.__status = status self.__formerStatus = formerStatus self.__reason = reason self.__siteType = siteType self.__serviceType = serviceType self.__resourceType = resourceType self.__useNewRes = useNewRes ################################################################################ def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ polToEval = self.iGetter.getInfoToApply( ('policy', 'policyType'), granularity=self.__granularity, statusType=self.__statusType, status=self.__status, formerStatus=self.__formerStatus, siteType=self.__siteType, serviceType=self.__serviceType, resourceType=self.__resourceType, useNewRes=self.__useNewRes) policyType = polToEval['PolicyType'] # type: generator if policyIn: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = policyIn.evaluate() else: singlePolicyResults = self._invocation(self.__granularity, self.__name, self.__status, policyIn, argsIn, polToEval['Policies']) policyCombinedResults = self._policyCombination(singlePolicyResults) if policyCombinedResults == {}: policyCombinedResults['Action'] = False policyCombinedResults['Reason'] = 'No policy results' policyCombinedResults['PolicyType'] = policyType if policyCombinedResults.has_key('Status'): newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy newPolicyType = self.iGetter.getNewPolicyType( self.__granularity, newstatus) policyType = set(policyType) & set(newPolicyType) policyCombinedResults['Action'] = True else: # Policies does not satisfy policyCombinedResults['Action'] = False policyCombinedResults['PolicyType'] = policyType return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': policyCombinedResults } ################################################################################ def _invocation(self, granularity, name, status, policy, args, policies): ''' One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. Always returns a list, possibly empty. ''' policyResults = [] for pol in policies: pName = pol['Name'] pModule = pol['Module'] extraArgs = pol['args'] commandIn = pol['commandIn'] res = self.pCaller.policyInvocation(granularity=granularity, name=name, status=status, policy=policy, args=args, pName=pName, pModule=pModule, extraArgs=extraArgs, commandIn=commandIn) # If res is empty, return immediately if not res: return policyResults if not res.has_key('Status'): print('\n\n Policy result ' + str(res) + ' does not return "Status"\n\n') raise TypeError # Else if res['Status'] == 'Unknown': res = self.__useOldPolicyRes(name=name, policyName=pName) if res['Status'] not in ('Error', 'Unknown'): policyResults.append(res) else: gLogger.warn(res) return policyResults ################################################################################ def _policyCombination(self, pol_results): ''' INPUT: list type OUTPUT: dict type * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result ''' if pol_results == []: return {} pol_results.sort(key=Status.value_of_policy) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions _prio, access_list, gofun = Status.statesInfo[self.__status] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for polRes in pol_results: if Status.value_of_policy(polRes) in access_list: newStatus = Status.value_of_policy(polRes) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun(access_list) return { 'Status': Status.status_of_value(newStatus), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = Status.value_of_policy(pol_results[0]) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = Status.value_of_policy(pol_results[0]) # At this point, a new status has been chosen. newStatus is an # integer. worstResults = [ p for p in pol_results if Status.value_of_policy(p) == newStatus ] # Concatenate reasons def getReason(pol): try: res = pol['Reason'] except KeyError: res = '' return res worstResultsReasons = [getReason(p) for p in worstResults] def catRes(xVal, yVal): ''' Concatenate xVal and yVal. ''' if xVal and yVal: return xVal + ' |###| ' + yVal elif xVal or yVal: if xVal: return xVal else: return yVal else: return '' concatenatedRes = reduce(catRes, worstResultsReasons, '') # Handle EndDate endDatePolicies = [p for p in worstResults if p.has_key('EndDate')] # Building and returning result res = {} res['Status'] = Status.status_of_value(newStatus) if concatenatedRes != '': res['Reason'] = concatenatedRes if endDatePolicies != []: res['EndDate'] = endDatePolicies[0]['EndDate'] return res ################################################################################ def __useOldPolicyRes(self, name, policyName): ''' Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} ''' res = self.clients['ResourceManagementClient'].getPolicyResult( name=name, policyName=policyName) if not res['OK']: return {'Status': 'Unknown'} res = res['Value'] if res == []: return {'Status': 'Unknown'} res = res[0] oldStatus = res[5] oldReason = res[6] lastCheckTime = res[8] if (lastCheckTime + datetime.timedelta(hours=2)) < datetime.datetime.utcnow(): return {'Status': 'Unknown'} result = {} result['Status'] = oldStatus result['Reason'] = oldReason result['OLD'] = True result['PolicyName'] = policyName return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class PDP( object ): """ PDP ( Policy Decision Point ) """ def __init__( self, clients = None ): """ Constructor. examples: >>> pdp = PDP( None ) >>> pdp1 = PDP( {} ) >>> pdp2 = PDP( { 'Client1' : Client1Object } ) :Parameters: **clients** - [ None, `dict` ] dictionary with Clients to be used in the Commands. If None, the Commands will create their own clients. """ # decision parameters used to match policies and actions self.decisionParams = None # Helpers to discover policies and RSS metadata in CS self.iGetter = InfoGetter() self.pCaller = PolicyCaller( clients ) # RSS State Machine, used to calculate most penalizing state while merging them self.rssMachine = RSSMachine( 'Unknown' ) self.log = gLogger.getSubLogger( 'PDP' ) def setup( self, decisionParams = None ): """ method that sanitizes the decisionParams and ensures that at least it has the keys in `standardParamsDict`. This will be relevant while doing the matching with the RSS Policies configuration in the CS. There is one key-value pair, `active` which is added on this method. This allows policies to be de-activated from the CS, changing their active matchParam to something else than `Active`. examples: >>> pdp.setup( None ) >>> self.decisionParams { 'element' : None, 'name' : None, ... } >>> pdp.setup( { 'element' : 'AnElement' } ) >>> self.decisionParams { 'element' : 'AnElement', 'name' : None, ... } >>> pdp.setup( { 'NonStandardKey' : 'Something' } ) >>> self.decisionParams { 'NonStandardKey' : 'Something', 'element' : None,... } :Parameters: **decisionParams** - [ None, `dict` ] dictionary with the parameters to be matched with the RSS Policies configuration in the CS. """ standardParamsDict = {'element' : None, 'name' : None, 'elementType' : None, 'statusType' : None, 'status' : None, 'reason' : None, 'tokenOwner' : None, # Last parameter allows policies to be de-activated 'active' : 'Active'} if decisionParams is not None: standardParamsDict.update( decisionParams ) if standardParamsDict['element'] is not None: self.log = gLogger.getSubLogger( 'PDP/%s' % standardParamsDict['element'] ) if standardParamsDict['name'] is not None: self.log = gLogger.getSubLogger( 'PDP/%s/%s' % ( standardParamsDict['element'], standardParamsDict['name'] ) ) self.log.verbose( "Setup - statusType: %s, status: %s" % ( standardParamsDict['statusType'], standardParamsDict['status'] ) ) self.decisionParams = standardParamsDict def takeDecision( self ): """ main PDP method which does all the work. If firstly finds all the policies defined in the CS that match <self.decisionParams> and runs them. Once it has all the singlePolicyResults, it combines them. Next step is action discovery: using a similar approach to the one used to discover the policies, but also taking into account the single policy results and their combined result, finds the actions to be triggered and returns. examples: >>> pdp.takeDecision()[ 'Value' ].keys() [ 'singlePolicyResults', 'policyCombinedResult', 'decisionParams' ] >>> pdp.takeDecision()[ 'Value' ][ 'singlePolicyResults' ] [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : { 'name' : 'AlwaysActiveForResource', 'type' : 'AlwaysActive', 'module' : 'AlwaysActivePolicy', 'description' : 'This is the AlwaysActive policy' 'command' : None, 'args' : {} } }, ... ] >>> pdp.takeDecision()[ 'Value' ][ 'policyCombinedResult' ] { 'Status' : 'Active', 'Reason' : 'blah ###', 'PolicyAction' : [ ( 'policyActionName1', 'policyActionType1' ), ... ] } :return: S_OK( { 'singlePolicyResults' : `list`, 'policyCombinedResult' : `dict`, 'decisionParams' : `dict` } ) / S_ERROR """ if self.decisionParams is None: return S_OK( {'singlePolicyResults' : [], 'policyCombinedResult' : {}, 'decisionParams' : self.decisionParams} ) self.log.verbose( "Taking decision" ) # Policies.................................................................. # Get policies that match self.decisionParams policiesThatApply = self.iGetter.getPoliciesThatApply( self.decisionParams ) if not policiesThatApply[ 'OK' ]: return policiesThatApply policiesThatApply = policiesThatApply[ 'Value' ] self.log.verbose( "Policies that apply: %s" % ', '.join( [po['name'] for po in policiesThatApply] ) ) # Evaluate policies singlePolicyResults = self._runPolicies( policiesThatApply ) if not singlePolicyResults[ 'OK' ]: return singlePolicyResults singlePolicyResults = singlePolicyResults[ 'Value' ] self.log.verbose( "Single policy results: %s" % singlePolicyResults ) # Combine policies and get most penalizing status ( see RSSMachine ) policyCombinedResults = self._combineSinglePolicyResults( singlePolicyResults ) if not policyCombinedResults[ 'OK' ]: return policyCombinedResults policyCombinedResults = policyCombinedResults[ 'Value' ] self.log.verbose( "Combined policy result: %s" % policyCombinedResults ) # Actions................................................................... policyActionsThatApply = self.iGetter.getPolicyActionsThatApply( self.decisionParams, singlePolicyResults, policyCombinedResults ) if not policyActionsThatApply[ 'OK' ]: return policyActionsThatApply policyActionsThatApply = policyActionsThatApply[ 'Value' ] self.log.verbose( "Policy actions that apply: %s" % ','.join( pata[0] for pata in policyActionsThatApply ) ) policyCombinedResults[ 'PolicyAction' ] = policyActionsThatApply return S_OK( {'singlePolicyResults' : singlePolicyResults, 'policyCombinedResult' : policyCombinedResults, 'decisionParams' : self.decisionParams} ) def _runPolicies( self, policies ): """ Given a list of policy dictionaries, loads them making use of the PolicyCaller and evaluates them. This method requires to have run setup previously. examples: >>> pdp._runPolicies([])[ 'Value' ] [] >>> policyDict = { 'name' : 'AlwaysActiveResource', 'type' : 'AlwaysActive', 'args' : None, 'description' : 'This is the AlwaysActive policy', 'module' : 'AlwaysActivePolicy', 'command' : None } >>> pdp._runPolicies([ policyDict, ... ] )[ 'Value' ] [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict }, ... ] :Parameters: **policies** - `list( dict )` list of dictionaries containing the policies selected to be run. Check the examples to get an idea of how the policy dictionaries look like. :return: S_OK() / S_ERROR """ policyInvocationResults = [] # Gets all valid status for RSS to avoid misconfigured policies returning statuses # that RSS does not understand. validStatus = self.rssMachine.getStates() for policyDict in policies: # Load and evaluate policy described in <policyDict> for element described # in <self.decisionParams> policyInvocationResult = self.pCaller.policyInvocation( self.decisionParams, policyDict ) if not policyInvocationResult[ 'OK' ]: # We should never enter this line ! Just in case there are policies # missconfigured ! _msg = 'runPolicies no OK: %s' % policyInvocationResult self.log.error( _msg ) return S_ERROR( _msg ) policyInvocationResult = policyInvocationResult[ 'Value' ] # Sanity Checks ( they should never happen ! ) if not 'Status' in policyInvocationResult: _msg = 'runPolicies (no Status): %s' % policyInvocationResult self.log.error( _msg ) return S_ERROR( _msg ) if not policyInvocationResult[ 'Status' ] in validStatus: _msg = 'runPolicies ( not valid status ) %s' % policyInvocationResult[ 'Status' ] self.log.error( _msg ) return S_ERROR( _msg ) if not 'Reason' in policyInvocationResult: _msg = 'runPolicies (no Reason): %s' % policyInvocationResult self.log.error( _msg ) return S_ERROR( _msg ) policyInvocationResults.append( policyInvocationResult ) return S_OK( policyInvocationResults ) def _combineSinglePolicyResults( self, singlePolicyRes ): """ method that merges all the policies results into a combined one, which will be the most penalizing status and the reasons of the single policy results that returned the same penalizing status. All the rest, are ignored. If there are no single policy results, it is returned `Unknown` state. While combining policies, the ones containing the option `doNotCombine` are ignored. examples: >>> pdp._combineSingePolicyResults( [] )[ 'Value' ] { 'Status' : 'Unknown', 'Reason' : 'No policy ..' } >>> pdp._combineSingePolicyResults( [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict } ] ) { 'Status' : 'Active', 'Reason' : 'blah' } >>> pdp._combineSingePolicyResults( [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict }, { 'Status' : 'Banned', 'Reason' : 'blah 2', 'Policy' : policyDict2 } ] ) { 'Status' : 'Banned', 'Reason' : 'blah 2' } >>> pdp._combineSingePolicyResults( [ { 'Status' : 'Active', 'Reason' : 'blah', 'Policy' : policyDict }, { 'Status' : 'Active', 'Reason' : 'blah 2', 'Policy' : policyDict2 } ] ) { 'Status' : 'Banned', 'Reason' : 'blah ### blah 2' } :Parameters: **singlePolicyRes** - `list( dict )` list with every single policy result to be combined ( see _runPolicy for more details ) :return: S_OK( dict( Status, Reason ) | S_ERROR """ # Dictionary to be returned policyCombined = { 'Status' : 'Unknown', 'Reason' : '' } # If there are no policyResults, we return Unknown if not singlePolicyRes: policyCombined[ 'Status' ] = 'Unknown' policyCombined[ 'Reason' ] = 'No policy applies to %(element)s, %(name)s, %(elementType)s' % self.decisionParams return S_OK( policyCombined ) # We set the rssMachine on the current state ( ensures it is a valid one ) # FIXME: probably this check can be done at takeDecission machineStatus = self.rssMachine.setState( self.decisionParams[ 'status' ] ) if not machineStatus[ 'OK' ]: return machineStatus # Discard all single policy results which belogs to policies that have set # the option `doNotCombine` in the CS policiesToCombine = self._findPoliciesToCombine( singlePolicyRes ) # Sort policy results using ther statuses by most restrictive ( lower level first ) self.rssMachine.orderPolicyResults( policiesToCombine ) # As they have been sorted by most restrictive status, the first one is going # to be our candidate new state. Let's ask the RSSMachine if it allows us to # make such transition. candidateState = policiesToCombine[ 0 ][ 'Status' ] nextState = self.rssMachine.getNextState( candidateState ) if not nextState[ 'OK' ]: return nextState nextState = nextState[ 'Value' ] # If the RssMachine does not accept the candidate, return forcing message if candidateState != nextState: policyCombined[ 'Status' ] = nextState policyCombined[ 'Reason' ] = 'RssMachine forced status %s to %s' % ( candidateState, nextState ) return S_OK( policyCombined ) # If the RssMachine accepts the candidate, just concatenate the reasons for policyRes in policiesToCombine: if policyRes[ 'Status' ] == nextState: policyCombined[ 'Reason' ] += '%s ###' % policyRes[ 'Reason' ] policyCombined[ 'Status' ] = nextState return S_OK( policyCombined ) def _findPoliciesToCombine( self, singlePolicyRes ): """ method that iterates over the single policy results and checks the CS configuration of the policies looking for the option 'doNotCombine'. If it is present, that single policy result is discarded. :Parameters: **singlePolicyRes** - `list( dict )` list with every single policy result to be combined ( see _runPolicy for more details ) :return: `list( dict )` """ # Get policies configuration from the CS. We want to exclude the policies that # have set the option `doNotCombine` from this process. policiesConfiguration = RssConfiguration.getPolicies() if not policiesConfiguration[ 'OK' ]: return policiesConfiguration policiesConfiguration = policiesConfiguration[ 'Value' ] # Function that let's us know if we should combine the result of a single policy # or not. def combinePolicy( policyResult ): # Extract policy name from the dictionary returned by PolicyCaller policyName = policyResult[ 'Policy' ][ 'name' ] try: # If doNotCombineResult is defined, the policy is not taken into account # to create the combined result. However, the single policy result remains _ = policiesConfiguration[ policyName ][ 'doNotCombineResult' ] return False except KeyError: return True # Make a list of policies of which we want to merge their results return [ policyResult for policyResult in singlePolicyRes if combinePolicy( policyResult ) ]
class ResourceStatus(object): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ __metaclass__ = DIRACSingleton def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None self.infoGetter = InfoGetter() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache('StorageElement', cacheLifeTime, self.__updateSECache) def getStorageElementStatus(self, elementName, statusType=None, default=None): """ Helper with dual access, tries to get information from the RSS for the given StorageElement, otherwise, it gets it from the CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'CERN-USER' : { 'ReadAccess': 'Active' } } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( { 'CERN-USER' : {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'}} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): # We do not apply defaults. If is not on the cache, S_ERROR is returned. return self.__getRSSStorageElementStatus(elementName, statusType) else: return self.__getCSStorageElementStatus(elementName, statusType, default) def setStorageElementStatus(self, elementName, statusType, status, reason=None, tokenOwner=None): """ Helper with dual access, tries set information in RSS and in CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'ReadAccess': 'Active' } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): return self.__setRSSStorageElementStatus(elementName, statusType, status, reason, tokenOwner) else: return self.__setCSStorageElementStatus(elementName, statusType, status) ################################################################################ def __updateSECache(self): """ Method used to update the StorageElementCache. """ meta = {'columns': ['Name', 'StatusType', 'Status']} rawCache = self.rssClient.selectStatusElement( 'Resource', 'Status', elementType='StorageElement', meta=meta) if not rawCache['OK']: return rawCache return S_OK(getCacheDictFromRawData(rawCache['Value'])) ################################################################################ def __getRSSStorageElementStatus(self, elementName, statusType): """ Gets from the cache or the RSS the StorageElements status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new StorageElement. The period between it is added to the DB and the changes are propagated to the cache will be inconsisten, but not dangerous. Just wait <cacheLifeTime> minutes. """ cacheMatch = self.seCache.match(elementName, statusType) self.log.debug('__getRSSStorageElementStatus') self.log.debug(cacheMatch) return cacheMatch def __getCSStorageElementStatus(self, elementName, statusType, default): """ Gets from the CS the StorageElements status """ cs_path = "/Resources/StorageElements" if not isinstance(elementName, list): elementName = [elementName] statuses = self.rssConfig.getConfigStatusType('StorageElement') result = {} for element in elementName: if statusType is not None: # Added Active by default res = gConfig.getOption( "%s/%s/%s" % (cs_path, element, statusType), 'Active') if res['OK'] and res['Value']: result[element] = {statusType: res['Value']} else: res = gConfig.getOptionsDict("%s/%s" % (cs_path, element)) if res['OK'] and res['Value']: elementStatuses = {} for elementStatusType, value in res['Value'].items(): if elementStatusType in statuses: elementStatuses[elementStatusType] = value # If there is no status defined in the CS, we add by default Read and # Write as Active. if elementStatuses == {}: elementStatuses = { 'ReadAccess': 'Active', 'WriteAccess': 'Active' } result[element] = elementStatuses if result: return S_OK(result) if default is not None: # sec check if statusType is None: statusType = 'none' defList = [[el, statusType, default] for el in elementName] return S_OK(getDictFromList(defList)) _msg = "StorageElement '%s', with statusType '%s' is unknown for CS." return S_ERROR(_msg % (elementName, statusType)) def __setRSSStorageElementStatus(self, elementName, statusType, status, reason, tokenOwner): """ Sets on the RSS the StorageElements status """ expiration = datetime.datetime.utcnow() + datetime.timedelta(days=1) self.seCache.acquireLock() try: res = self.rssClient.modifyStatusElement( 'Resource', 'Status', name=elementName, statusType=statusType, status=status, reason=reason, tokenOwner=tokenOwner, tokenExpiration=expiration) if res['OK']: self.seCache.refreshCache() if not res['OK']: _msg = 'Error updating StorageElement (%s,%s,%s)' % ( elementName, statusType, status) gLogger.warn('RSS: %s' % _msg) return res finally: # Release lock, no matter what. self.seCache.releaseLock() def __setCSStorageElementStatus(self, elementName, statusType, status): """ Sets on the CS the StorageElements status """ statuses = self.rssConfig.getConfigStatusType('StorageElement') if not statusType in statuses: gLogger.error("%s is not a valid statusType" % statusType) return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses)) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption("%s/%s/%s" % (cs_path, elementName, statusType), status) res = csAPI.commitChanges() if not res['OK']: gLogger.warn('CS: %s' % res['Message']) return res def __getMode(self): """ Get's flag defined ( or not ) on the RSSConfiguration. If defined as 1, we use RSS, if not, we use CS. """ res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False def isStorageElementAlwaysBanned(self, seName): """ Checks if the AlwaysBanned policy is applied to the SE given as parameter :param seName : string, name of the SE :returns S_OK(True/False) """ res = self.infoGetter.getPoliciesThatApply({'name': seName}) if not res['OK']: self.log.error( "isStorageElementAlwaysBanned: unable to get the information", res['Message']) return res isAlwaysBanned = 'AlwaysBanned' in [ policy['type'] for policy in res['Value'] ] return S_OK(isAlwaysBanned)
class PDP: """ PDP = Policy Decision Point. Used to invoke policies and to take decision based on the polict results combination. """ ############################################################################# def __init__(self, VOExtension, granularity = None, name = None, status = None, formerStatus = None, reason = None, siteType = None, serviceType = None, resourceType = None, useNewRes = False): """ PDP (Policy Decision Point) initialization :params: :attr:`VOExtension`: string - VO extension (e.g. 'LHCb') :attr:`granularity`: string - a ValidRes :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.VOExtension = VOExtension self.__granularity = assignOrRaise(granularity, ValidRes, InvalidRes, self, self.__init__) self.__name = name self.__status = assignOrRaise(status, ValidStatus, InvalidStatus, self, self.__init__) self.__formerStatus = assignOrRaise(formerStatus, ValidStatus, InvalidStatus, self, self.__init__) self.__reason = reason self.__siteType = assignOrRaise(siteType, ValidSiteType, InvalidSiteType, self, self.__init__) self.__serviceType = assignOrRaise(serviceType, ValidServiceType, InvalidServiceType, self, self.__init__) self.__resourceType = assignOrRaise(resourceType, ValidResourceType, InvalidResourceType, self, self.__init__) cc = CommandCaller() self.pc = PolicyCaller(cc) self.useNewRes = useNewRes self.args = None self.policy = None self.knownInfo = None self.ig = None ############################################################################# def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ self.args = argsIn self.policy = policyIn self.knownInfo = knownInfo self.ig = InfoGetter(self.VOExtension) EVAL = self.ig.getInfoToApply(('policy', 'policyType'), granularity = self.__granularity, status = self.__status, formerStatus = self.__formerStatus, siteType = self.__siteType, serviceType = self.__serviceType, resourceType = self.__resourceType, useNewRes = self.useNewRes) policyCombinedResultsList = [] for policyGroup in EVAL: policyType = policyGroup['PolicyType'] if self.policy is not None: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = self.policy.evaluate() else: if policyGroup['Policies'] is None: return {'SinglePolicyResults' : [], 'PolicyCombinedResult' : [{'PolicyType': policyType, 'Action': False, 'Reason':'No policy results'}]} else: singlePolicyResults = self._invocation(self.VOExtension, self.__granularity, self.__name, self.__status, self.policy, self.args, policyGroup['Policies']) policyCombinedResults = self._policyCombination(singlePolicyResults) if not policyCombinedResults: return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': [] } # # policy results communication # newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy reason = policyCombinedResults['Reason'] newPolicyType = self.ig.getNewPolicyType(self.__granularity, newstatus) for npt in newPolicyType: if npt not in policyType: policyType.append(npt) decision = { 'PolicyType': policyType, 'Action': True, 'Status': newstatus, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] policyCombinedResultsList.append(decision) else: # Policies does not satisfy reason = policyCombinedResults['Reason'] decision = { 'PolicyType': policyType, 'Action': False, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] policyCombinedResultsList.append(decision) res = { 'SinglePolicyResults' : singlePolicyResults, 'PolicyCombinedResult' : policyCombinedResultsList } return res ############################################################################# def _invocation(self, VOExtension, granularity, name, status, policy, args, policies): """ One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. """ policyResults = [] for p in policies: pName = p['Name'] pModule = p['Module'] extraArgs = p['args'] commandIn = p['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity = granularity, name = name, status = status, policy = policy, args = args, pName = pName, pModule = pModule, extraArgs = extraArgs, commandIn = commandIn) # If res is empty, return immediately if not res: return policyResults if not res.has_key('Status'): print("\n\n Policy result " + str(res) + " does not return 'Status'\n\n") raise TypeError # Else if res['Status'] == 'Unknown': res = self.__useOldPolicyRes(name = name, policyName = pName) if res['Status'] == 'NeedConfirmation': pName = p['ConfirmationPolicy'] triggeredPolicy = self.ig.C_Policies[pName] pModule = triggeredPolicy['module'] extraArgs = triggeredPolicy['args'] commandIn = triggeredPolicy['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity = granularity, name = name, status = status, policy = policy, args = args, pName = pName, pModule = pModule, extraArgs = extraArgs, commandIn = commandIn) if res['Status'] not in ('Error', 'Unknown', 'NeedConfirmation'): policyResults.append(res) return policyResults ############################################################################# def _policyCombination(self, policies): """ * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result """ if policies == []: return {} policies.sort(key=value_of_policy) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions prio, access_list, gofun = statesInfo[self.__status] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for p in policies: if value_of_policy(p) in access_list: newStatus = value_of_policy(p) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun(access_list) return { 'Status': status_of_value(newStatus), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = value_of_policy(policies[0]) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = value_of_policy(policies[0]) # At this point, a new status has been chosen. newStatus is an # integer. worstPolicies = [p for p in policies if value_of_policy(p) == newStatus] # Concatenate reasons def getReason(p): try: res = p['Reason'] except KeyError: res = '' return res worstPoliciesReasons = [getReason(p) for p in worstPolicies] def catRes(x, y): if x and y : return x + ' |###| ' + y elif x or y: if x: return x else: return y else : return '' concatenatedRes = reduce(catRes, worstPoliciesReasons, '') # Handle EndDate endDatePolicies = [p for p in worstPolicies if p.has_key('EndDate')] # Building and returning result res = {} res['Status'] = status_of_value(newStatus) if concatenatedRes != '': res['Reason'] = concatenatedRes if endDatePolicies != []: res['EndDate'] = endDatePolicies[0]['EndDate'] return res ############################################################################# def __useOldPolicyRes(self, name, policyName): """ Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} """ from DIRAC.Core.DISET.RPCClient import RPCClient rsS = RPCClient("ResourceStatus/ResourceManagement") res = rsS.getPolicyRes(name, policyName, True) if not res['OK']: raise RSSException, where(self, self.__useOldPolicyRes) + ' Could not get a policy result' res = res['Value'] if res == []: return {'Status':'Unknown'} oldStatus = res[0] oldReason = res[1] lastCheckTime = res[2] if ( lastCheckTime + datetime.timedelta(hours = 2) ) < datetime.datetime.utcnow(): return {'Status':'Unknown'} result = {} result['Status'] = oldStatus result['Reason'] = oldReason result['OLD'] = True result['PolicyName'] = policyName return result
def testGetInfoToApply(self): ig = InfoGetter("LHCb") for g in ValidRes: for s in ValidStatus: for site_t in ValidSiteType: for service_t in ValidServiceType: if g in ("Site", "Sites"): panel = "Site_Panel" if g in ("Service", "Services"): if service_t == "Storage": panel = "Service_Storage_Panel" if service_t == "Computing": panel = "Service_Computing_Panel" if service_t == "VO-BOX": panel = "Service_VO-BOX_Panel" if service_t == "VOMS": panel = "Service_VOMS_Panel" if g in ("Resource", "Resources"): panel = "Resource_Panel" if g in ("StorageElementRead", "StorageElementsRead"): panel = "SE_Panel" if g in ("StorageElementWrite", "StorageElementsWrite"): panel = "SE_Panel" for resource_t in ValidResourceType: ## Testing the policyType (__getPolTypes) part res = ig.getInfoToApply(("policyType",), g, None, s, None, site_t, service_t, resource_t) for p_res in res["PolicyType"]: self.assert_(p_res in CS.getTypedDictRootedAt("PolicyTypes").keys()) for useNewRes in (False, True): ## Testing the policy (__getPolToEval) part res = ig.getInfoToApply( ("policy",), g, None, s, None, site_t, service_t, resource_t, useNewRes ) pModuleList = [] for k in self.configModule.Policies.keys(): try: if self.configModule.Policies[k]["module"] not in pModuleList: pModuleList.append(self.configModule.Policies[k]["module"]) except KeyError: pass for p_res in res["Policies"]: # All __getPolToEval results... self.assertTrue(p_res["Name"] in CS.getTypedDictRootedAt("Policies")) # self.assertTrue(p_res['Module'] in pModuleList) if useNewRes is False: self.assertEqual( p_res["commandIn"], self.configModule.Policies[p_res["Name"]]["commandIn"] ) self.assertEqual( p_res["args"], self.configModule.Policies[p_res["Name"]]["args"] ) else: try: self.assertEqual( p_res["commandIn"], self.configModule.Policies[p_res["Name"]]["commandInNewRes"], ) except KeyError: self.assertEqual( p_res["commandIn"], self.configModule.Policies[p_res["Name"]]["commandIn"], ) try: self.assertEqual( p_res["args"], self.configModule.Policies[p_res["Name"]]["argsNewRes"] ) except KeyError: self.assertEqual( p_res["args"], self.configModule.Policies[p_res["Name"]]["args"] ) res = ig.getInfoToApply( ("panel_info",), g, None, s, None, site_t, service_t, resource_t, useNewRes ) for p_res in res["Info"]: # if 'JobsEfficiencySimple' in p_res.keys(): # print useNewRes, p_res for p_name in p_res.keys(): self.assert_(p_name in self.configModule.Policies.keys()) if isinstance(p_res[p_name], list): for i in range(len(p_res[p_name])): for k in p_res[p_name][i].keys(): if useNewRes: try: self.assertEqual( p_res[p_name][i][k]["CommandIn"], self.configModule.Policies[p_name][panel][i][k][ "CommandInNewRes" ], ) except KeyError: self.assertEqual( p_res[p_name][i][k]["CommandIn"], self.configModule.Policies[p_name][panel][i][k][ "CommandIn" ], ) except TypeError: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) try: self.assertEqual( p_res[p_name][i][k]["args"], self.configModule.Policies[p_name][panel][i][k][ "argsNewRes" ], ) except KeyError: self.assertEqual( p_res[p_name][i][k]["args"], self.configModule.Policies[p_name][panel][i][k]["args"], ) except TypeError: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) else: try: self.assertEqual( p_res[p_name][i][k]["CommandIn"], self.configModule.Policies[p_name][panel][i][k][ "CommandIn" ], ) except: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) try: self.assertEqual( p_res[p_name][i][k]["args"], self.configModule.Policies[p_name][panel][i][k]["args"], ) except: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) else: self.assertEqual(p_res[p_name], self.configModule.Policies[p_name][panel])
class ResourceStatus( object ): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ __metaclass__ = DIRACSingleton def __init__( self ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None self.infoGetter = InfoGetter() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( 'StorageElement', cacheLifeTime, self.__updateSECache ) def getStorageElementStatus( self, elementName, statusType = None, default = None ): """ Helper with dual access, tries to get information from the RSS for the given StorageElement, otherwise, it gets it from the CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'CERN-USER' : { 'ReadAccess': 'Active' } } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( { 'CERN-USER' : {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'}} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): # We do not apply defaults. If is not on the cache, S_ERROR is returned. return self.__getRSSStorageElementStatus( elementName, statusType ) else: return self.__getCSStorageElementStatus( elementName, statusType, default ) def setStorageElementStatus( self, elementName, statusType, status, reason = None, tokenOwner = None ): """ Helper with dual access, tries set information in RSS and in CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'ReadAccess': 'Active' } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): return self.__setRSSStorageElementStatus( elementName, statusType, status, reason, tokenOwner ) else: return self.__setCSStorageElementStatus( elementName, statusType, status ) ################################################################################ def __updateSECache( self ): """ Method used to update the StorageElementCache. It will try 5 times to contact the RSS before giving up """ meta = { 'columns' : [ 'Name', 'StatusType', 'Status' ] } for ti in range( 5 ): rawCache = self.rssClient.selectStatusElement( 'Resource', 'Status', elementType = 'StorageElement', meta = meta ) if rawCache['OK']: break self.log.warn( "Can't get SE status", rawCache['Message'] + "; trial %d" % ti ) sleep( math.pow( ti, 2 ) ) self.rssClient = ResourceStatusClient() if not rawCache[ 'OK' ]: return rawCache return S_OK( getCacheDictFromRawData( rawCache[ 'Value' ] ) ) ################################################################################ def __getRSSStorageElementStatus( self, elementName, statusType ): """ Gets from the cache or the RSS the StorageElements status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new StorageElement. The period between it is added to the DB and the changes are propagated to the cache will be inconsisten, but not dangerous. Just wait <cacheLifeTime> minutes. """ cacheMatch = self.seCache.match( elementName, statusType ) self.log.debug( '__getRSSStorageElementStatus' ) self.log.debug( cacheMatch ) return cacheMatch def __getCSStorageElementStatus( self, elementName, statusType, default ): """ Gets from the CS the StorageElements status """ cs_path = "/Resources/StorageElements" if not isinstance( elementName, list ): elementName = [ elementName ] statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) result = {} for element in elementName: if statusType is not None: # Added Active by default res = gConfig.getValue( "%s/%s/%s" % ( cs_path, element, statusType ), 'Active' ) result[element] = {statusType: res} else: res = gConfig.getOptionsDict( "%s/%s" % ( cs_path, element ) ) if res[ 'OK' ] and res[ 'Value' ]: elementStatuses = {} for elementStatusType, value in res[ 'Value' ].items(): if elementStatusType in statuses: elementStatuses[ elementStatusType ] = value # If there is no status defined in the CS, we add by default Read and # Write as Active. if elementStatuses == {}: elementStatuses = { 'ReadAccess' : 'Active', 'WriteAccess' : 'Active' } result[ element ] = elementStatuses if result: return S_OK( result ) if default is not None: # sec check if statusType is None: statusType = 'none' defList = [ [ el, statusType, default ] for el in elementName ] return S_OK( getDictFromList( defList ) ) _msg = "StorageElement '%s', with statusType '%s' is unknown for CS." return S_ERROR( _msg % ( elementName, statusType ) ) def __setRSSStorageElementStatus( self, elementName, statusType, status, reason, tokenOwner ): """ Sets on the RSS the StorageElements status """ expiration = datetime.datetime.utcnow() + datetime.timedelta( days = 1 ) self.seCache.acquireLock() try: res = self.rssClient.modifyStatusElement( 'Resource', 'Status', name = elementName, statusType = statusType, status = status, reason = reason, tokenOwner = tokenOwner, tokenExpiration = expiration ) if res[ 'OK' ]: self.seCache.refreshCache() if not res[ 'OK' ]: _msg = 'Error updating StorageElement (%s,%s,%s)' % ( elementName, statusType, status ) gLogger.warn( 'RSS: %s' % _msg ) return res finally: # Release lock, no matter what. self.seCache.releaseLock() def __setCSStorageElementStatus( self, elementName, statusType, status ): """ Sets on the CS the StorageElements status """ statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) if not statusType in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res def __getMode( self ): """ Get's flag defined ( or not ) on the RSSConfiguration. If defined as 1, we use RSS, if not, we use CS. """ res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False def isStorageElementAlwaysBanned( self, seName, statusType ): """ Checks if the AlwaysBanned policy is applied to the SE given as parameter :param seName : string, name of the SE :param statusType : ReadAcces, WriteAccess, RemoveAccess, CheckAccess :returns S_OK(True/False) """ res = self.infoGetter.getPoliciesThatApply( {'name' : seName, 'statusType' : statusType} ) if not res['OK']: self.log.error( "isStorageElementAlwaysBanned: unable to get the information", res['Message'] ) return res isAlwaysBanned = 'AlwaysBanned' in [policy['type'] for policy in res['Value']] return S_OK( isAlwaysBanned )
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None, WMSAdminIn = None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool( 2, 5 ) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes = False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, Utils.where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'], resourceName = name)[0][0] except IndexError: return "%s does not exist!" %name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'], storageElementName = name)[0][0] except IndexError: return "%s does not exist!" %name paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description'] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status, formerStatus = formerStatus, siteType = siteType, serviceType = serviceType, resourceType = resourceType, useNewRes = useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt(self.getInfoForPanel, args = (info, granularityForPanel, nameForPanel) ) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]} else: pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format_ = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what) info_res[format_] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, Utils.where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = { name : { 'RSSStatus': RSSStatus} } return status ############################################################################# def _getInfo(self, granularity, name, format_, what): if format_ == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName, serviceName = serviceName, serviceType = serviceType, resourceName = resourceName, storageElementName = storageElementName, gridSiteName = gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList(granularity, siteName = siteName, serviceName = serviceName, resourceName = resourceName, storageElementName = storageElementName) if res == []: return False else: return True
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*']) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes=False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList( 'Resource', ['ResourceType'], resourceName=name)[0][0] except IndexError: return "%s does not exist!" % name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList( 'StorageElement', ['SiteType'], storageElementName=name)[0][0] except IndexError: return "%s does not exist!" % name paramNames = [ 'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description' ] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status=status, formerStatus=formerStatus, siteType=siteType, serviceType=serviceType, resourceType=resourceType, useNewRes=useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel][ 'DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel][ 'RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt( self.getInfoForPanel, args=(info, granularityForPanel, nameForPanel)) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy][ 'Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]} else: pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format, what) info_res[format] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name: { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = {name: {'RSSStatus': RSSStatus}} return status ############################################################################# def _getInfo(self, granularity, name, format, what): if format == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList( gran, paramsList=paramsL, siteName=siteName, serviceName=serviceName, serviceType=serviceType, resourceName=resourceName, storageElementName=storageElementName, gridSiteName=gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList( granularity, siteName=siteName, serviceName=serviceName, resourceName=resourceName, storageElementName=storageElementName) if res == []: return False else: return True
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__(self, clients): ''' Constructor. Defines members that will be used later on. ''' self.pCaller = PolicyCaller(clients=clients) self.iGetter = InfoGetter() self.decissionParams = {} self.rssMachine = RSSMachine('Unknown') def setup(self, decissionParams=None): standardParamsDict = { 'element': None, 'name': None, 'elementType': None, 'statusType': None, 'status': None, 'reason': None, 'tokenOwner': None, # Last parameter allows policies to be deactivated 'active': 'Active' } if decissionParams is not None: standardParamsDict.update(decissionParams) self.decissionParams = standardParamsDict ################################################################################ def takeDecision( self): #, policyIn = None, argsIn = None, knownInfo = None ): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason #'EndDate: datetime.datetime (in a string)} """ policiesThatApply = self.iGetter.getPoliciesThatApply( self.decissionParams) if not policiesThatApply['OK']: return policiesThatApply policiesThatApply = policiesThatApply['Value'] singlePolicyResults = self._runPolicies(policiesThatApply) if not singlePolicyResults['OK']: return singlePolicyResults singlePolicyResults = singlePolicyResults['Value'] policyCombinedResults = self._combineSinglePolicyResults( singlePolicyResults) if not policyCombinedResults['OK']: return policyCombinedResults policyCombinedResults = policyCombinedResults['Value'] policyActionsThatApply = self.iGetter.getPolicyActionsThatApply( self.decissionParams, singlePolicyResults, policyCombinedResults) if not policyActionsThatApply['OK']: return policyActionsThatApply policyActionsThatApply = policyActionsThatApply['Value'] policyCombinedResults['PolicyAction'] = policyActionsThatApply return S_OK({ 'singlePolicyResults': singlePolicyResults, 'policyCombinedResult': policyCombinedResults, 'decissionParams': self.decissionParams }) ################################################################################ def _runPolicies(self, policies, decissionParams=None): if decissionParams is None: decissionParams = self.decissionParams validStatus = RssConfiguration.getValidStatus() if not validStatus['OK']: return validStatus validStatus = validStatus['Value'] policyInvocationResults = [] for policyDict in policies: policyInvocationResult = self.pCaller.policyInvocation( decissionParams, policyDict) if not policyInvocationResult['OK']: # We should never enter this line ! Just in case there are policies # missconfigured ! _msg = 'runPolicies no OK: %s' % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) policyInvocationResult = policyInvocationResult['Value'] if not 'Status' in policyInvocationResult: _msg = 'runPolicies (no Status): %s' % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) if not policyInvocationResult['Status'] in validStatus: _msg = 'runPolicies ( not valid status ) %s' % policyInvocationResult[ 'Status'] gLogger.error(_msg) return S_ERROR(_msg) if not 'Reason' in policyInvocationResult: _msg = 'runPolicies (no Reason): %s' % policyInvocationResult gLogger.error(_msg) return S_ERROR(_msg) policyInvocationResults.append(policyInvocationResult) return S_OK(policyInvocationResults) ################################################################################ def _combineSinglePolicyResults(self, singlePolicyRes): ''' singlePolicyRes = [ { 'State' : X, 'Reason' : Y, ... }, ... ] If there are no policyResults, returns Unknown as there are no policies to apply. Order elements in list by state, being the lowest the most restrictive one in the hierarchy. ''' # Dictionary to be returned policyCombined = {'Status': None, 'Reason': ''} # If there are no policyResults, we return Unknown if not singlePolicyRes: _msgTuple = (self.decissionParams['element'], self.decissionParams['name'], self.decissionParams['elementType']) policyCombined['Status'] = 'Unknown' policyCombined[ 'Reason'] = 'No policy applies to %s, %s, %s' % _msgTuple return S_OK(policyCombined) # We set the rssMachine on the current state machineStatus = self.rssMachine.setState( self.decissionParams['status']) if not machineStatus['OK']: return machineStatus # Order statuses by most restrictive ( lower level first ) self.rssMachine.orderPolicyResults(singlePolicyRes) #policyResults = self.rssMachine.orderPolicyResults( singlePolicyRes ) # Get according to the RssMachine the next state, given a candidate candidateState = singlePolicyRes[0]['Status'] nextState = self.rssMachine.getNextState(candidateState) if not nextState['OK']: return nextState nextState = nextState['Value'] # If the RssMachine does not accept the candidate, return forcing message if candidateState != nextState: policyCombined['Status'] = nextState policyCombined['Reason'] = 'RssMachine forced status %s to %s' % ( candidateState, nextState) return S_OK(policyCombined) # If the RssMachine accepts the candidate, just concatenate the reasons for policyRes in singlePolicyRes: if policyRes['Status'] == nextState: policyCombined['Reason'] += '%s ###' % policyRes['Reason'] policyCombined['Status'] = nextState return S_OK(policyCombined) ################################################################################ # def __useOldPolicyRes( self, name, policyName ): # ''' # Use the RSS Service to get an old policy result. # If such result is older than 2 hours, it returns {'Status':'Unknown'} # ''' # res = self.clients[ 'ResourceManagementClient' ].getPolicyResult( name = name, policyName = policyName ) # # if not res[ 'OK' ]: # return { 'Status' : 'Unknown' } # # res = res[ 'Value' ] # # if res == []: # return { 'Status' : 'Unknown' } # # res = res[ 0 ] # # oldStatus = res[ 5 ] # oldReason = res[ 6 ] # lastCheckTime = res[ 8 ] # # if ( lastCheckTime + datetime.timedelta(hours = 2) ) < datetime.datetime.utcnow(): # return { 'Status' : 'Unknown' } # # result = {} # # result[ 'Status' ] = oldStatus # result[ 'Reason' ] = oldReason # result[ 'OLD' ] = True # result[ 'PolicyName' ] = policyName # # return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ self.args = argsIn self.policy = policyIn self.knownInfo = knownInfo self.ig = InfoGetter(self.VOExtension) EVAL = self.ig.getInfoToApply(('policy', 'policyType'), granularity = self.__granularity, status = self.__status, formerStatus = self.__formerStatus, siteType = self.__siteType, serviceType = self.__serviceType, resourceType = self.__resourceType, useNewRes = self.useNewRes) policyCombinedResultsList = [] for policyGroup in EVAL: policyType = policyGroup['PolicyType'] if self.policy is not None: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = self.policy.evaluate() else: if policyGroup['Policies'] is None: return {'SinglePolicyResults' : [], 'PolicyCombinedResult' : [{'PolicyType': policyType, 'Action': False, 'Reason':'No policy results'}]} else: singlePolicyResults = self._invocation(self.VOExtension, self.__granularity, self.__name, self.__status, self.policy, self.args, policyGroup['Policies']) policyCombinedResults = self._policyCombination(singlePolicyResults) if not policyCombinedResults: return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': [] } # # policy results communication # newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy reason = policyCombinedResults['Reason'] newPolicyType = self.ig.getNewPolicyType(self.__granularity, newstatus) for npt in newPolicyType: if npt not in policyType: policyType.append(npt) decision = { 'PolicyType': policyType, 'Action': True, 'Status': newstatus, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] policyCombinedResultsList.append(decision) else: # Policies does not satisfy reason = policyCombinedResults['Reason'] decision = { 'PolicyType': policyType, 'Action': False, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] policyCombinedResultsList.append(decision) res = { 'SinglePolicyResults' : singlePolicyResults, 'PolicyCombinedResult' : policyCombinedResultsList } return res
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__( self, **clients ): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller( cc, **clients ) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None def setup( self, granularity = None, name = None, statusType = None, status = None, formerStatus = None, reason = None, siteType = None, serviceType = None, resourceType = None, useNewRes = False ): """ PDP (Policy Decision Point) initialization :params: :attr:`granularity`: string - a ValidElement :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.__granularity = granularity self.__name = name self.__statusType = statusType self.__status = status self.__formerStatus = formerStatus self.__reason = reason self.__siteType = siteType self.__serviceType = serviceType self.__resourceType = resourceType self.__useNewRes = useNewRes ################################################################################ def takeDecision( self, policyIn = None, argsIn = None, knownInfo = None ): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ polToEval = self.iGetter.getInfoToApply( ( 'policy', 'policyType' ), granularity = self.__granularity, statusType = self.__statusType, status = self.__status, formerStatus = self.__formerStatus, siteType = self.__siteType, serviceType = self.__serviceType, resourceType = self.__resourceType, useNewRes = self.__useNewRes ) policyType = polToEval[ 'PolicyType' ] # type: generator if policyIn: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = policyIn.evaluate() else: singlePolicyResults = self._invocation( self.__granularity, self.__name, self.__status, policyIn, argsIn, polToEval['Policies'] ) policyCombinedResults = self._policyCombination( singlePolicyResults ) if policyCombinedResults == {}: policyCombinedResults[ 'Action' ] = False policyCombinedResults[ 'Reason' ] = 'No policy results' policyCombinedResults[ 'PolicyType' ] = policyType if policyCombinedResults.has_key( 'Status' ): newstatus = policyCombinedResults[ 'Status' ] if newstatus != self.__status: # Policies satisfy newPolicyType = self.iGetter.getNewPolicyType( self.__granularity, newstatus ) policyType = set( policyType ) & set( newPolicyType ) policyCombinedResults[ 'Action' ] = True else: # Policies does not satisfy policyCombinedResults[ 'Action' ] = False policyCombinedResults[ 'PolicyType' ] = policyType return { 'SinglePolicyResults' : singlePolicyResults, 'PolicyCombinedResult' : policyCombinedResults } ################################################################################ def _invocation( self, granularity, name, status, policy, args, policies ): ''' One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. Always returns a list, possibly empty. ''' policyResults = [] for pol in policies: pName = pol[ 'Name' ] pModule = pol[ 'Module' ] extraArgs = pol[ 'args' ] commandIn = pol[ 'commandIn' ] res = self.pCaller.policyInvocation( granularity = granularity, name = name, status = status, policy = policy, args = args, pName = pName, pModule = pModule, extraArgs = extraArgs, commandIn = commandIn ) # If res is empty, return immediately if not res: return policyResults if not res.has_key( 'Status' ): print('\n\n Policy result ' + str(res) + ' does not return "Status"\n\n') raise TypeError # Else if res[ 'Status' ] == 'Unknown': res = self.__useOldPolicyRes( name = name, policyName = pName ) if res[ 'Status' ] not in ( 'Error', 'Unknown' ): policyResults.append( res ) else: gLogger.warn( res ) return policyResults ################################################################################ def _policyCombination( self, pol_results ): ''' INPUT: list type OUTPUT: dict type * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result ''' if pol_results == []: return {} pol_results.sort( key=Status.value_of_policy ) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions _prio, access_list, gofun = Status.statesInfo[ self.__status ] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for polRes in pol_results: if Status.value_of_policy( polRes ) in access_list: newStatus = Status.value_of_policy( polRes ) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun( access_list ) return { 'Status': Status.status_of_value( newStatus ), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = Status.value_of_policy( pol_results[ 0 ] ) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = Status.value_of_policy( pol_results[ 0 ] ) # At this point, a new status has been chosen. newStatus is an # integer. worstResults = [ p for p in pol_results if Status.value_of_policy( p ) == newStatus ] # Concatenate reasons def getReason( pol ): try: res = pol[ 'Reason' ] except KeyError: res = '' return res worstResultsReasons = [ getReason( p ) for p in worstResults ] def catRes( xVal, yVal ): ''' Concatenate xVal and yVal. ''' if xVal and yVal : return xVal + ' |###| ' + yVal elif xVal or yVal: if xVal: return xVal else: return yVal else: return '' concatenatedRes = reduce( catRes, worstResultsReasons, '' ) # Handle EndDate endDatePolicies = [ p for p in worstResults if p.has_key( 'EndDate' ) ] # Building and returning result res = {} res[ 'Status' ] = Status.status_of_value( newStatus ) if concatenatedRes != '': res[ 'Reason' ] = concatenatedRes if endDatePolicies != []: res[ 'EndDate' ] = endDatePolicies[ 0 ][ 'EndDate' ] return res ################################################################################ def __useOldPolicyRes( self, name, policyName ): ''' Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} ''' res = self.clients[ 'ResourceManagementClient' ].getPolicyResult( name = name, policyName = policyName ) if not res[ 'OK' ]: return { 'Status' : 'Unknown' } res = res[ 'Value' ] if res == []: return { 'Status' : 'Unknown' } res = res[ 0 ] oldStatus = res[ 5 ] oldReason = res[ 6 ] lastCheckTime = res[ 8 ] if ( lastCheckTime + datetime.timedelta(hours = 2) ) < datetime.datetime.utcnow(): return { 'Status' : 'Unknown' } result = {} result[ 'Status' ] = oldStatus result[ 'Reason' ] = oldReason result[ 'OLD' ] = True result[ 'PolicyName' ] = policyName return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF