class PDP: """ PDP = Policy Decision Point. Used to invoke policies and to take decision based on the polict results combination. """ ############################################################################# def __init__(self, VOExtension, granularity = None, name = None, status = None, formerStatus = None, reason = None, siteType = None, serviceType = None, resourceType = None, useNewRes = False): """ PDP (Policy Decision Point) initialization :params: :attr:`VOExtension`: string - VO extension (e.g. 'LHCb') :attr:`granularity`: string - a ValidRes :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.VOExtension = VOExtension self.__granularity = assignOrRaise(granularity, ValidRes, InvalidRes, self, self.__init__) self.__name = name self.__status = assignOrRaise(status, ValidStatus, InvalidStatus, self, self.__init__) self.__formerStatus = assignOrRaise(formerStatus, ValidStatus, InvalidStatus, self, self.__init__) self.__reason = reason self.__siteType = assignOrRaise(siteType, ValidSiteType, InvalidSiteType, self, self.__init__) self.__serviceType = assignOrRaise(serviceType, ValidServiceType, InvalidServiceType, self, self.__init__) self.__resourceType = assignOrRaise(resourceType, ValidResourceType, InvalidResourceType, self, self.__init__) cc = CommandCaller() self.pc = PolicyCaller(cc) self.useNewRes = useNewRes self.args = None self.policy = None self.knownInfo = None self.ig = None ############################################################################# def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ self.args = argsIn self.policy = policyIn self.knownInfo = knownInfo self.ig = InfoGetter(self.VOExtension) EVAL = self.ig.getInfoToApply(('policy', 'policyType'), granularity = self.__granularity, status = self.__status, formerStatus = self.__formerStatus, siteType = self.__siteType, serviceType = self.__serviceType, resourceType = self.__resourceType, useNewRes = self.useNewRes) policyCombinedResultsList = [] for policyGroup in EVAL: policyType = policyGroup['PolicyType'] if self.policy is not None: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = self.policy.evaluate() else: if policyGroup['Policies'] is None: return {'SinglePolicyResults' : [], 'PolicyCombinedResult' : [{'PolicyType': policyType, 'Action': False, 'Reason':'No policy results'}]} else: singlePolicyResults = self._invocation(self.VOExtension, self.__granularity, self.__name, self.__status, self.policy, self.args, policyGroup['Policies']) policyCombinedResults = self._policyCombination(singlePolicyResults) if not policyCombinedResults: return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': [] } # # policy results communication # newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy reason = policyCombinedResults['Reason'] newPolicyType = self.ig.getNewPolicyType(self.__granularity, newstatus) for npt in newPolicyType: if npt not in policyType: policyType.append(npt) decision = { 'PolicyType': policyType, 'Action': True, 'Status': newstatus, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] policyCombinedResultsList.append(decision) else: # Policies does not satisfy reason = policyCombinedResults['Reason'] decision = { 'PolicyType': policyType, 'Action': False, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] policyCombinedResultsList.append(decision) res = { 'SinglePolicyResults' : singlePolicyResults, 'PolicyCombinedResult' : policyCombinedResultsList } return res ############################################################################# def _invocation(self, VOExtension, granularity, name, status, policy, args, policies): """ One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. """ policyResults = [] for p in policies: pName = p['Name'] pModule = p['Module'] extraArgs = p['args'] commandIn = p['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity = granularity, name = name, status = status, policy = policy, args = args, pName = pName, pModule = pModule, extraArgs = extraArgs, commandIn = commandIn) # If res is empty, return immediately if not res: return policyResults if not res.has_key('Status'): print("\n\n Policy result " + str(res) + " does not return 'Status'\n\n") raise TypeError # Else if res['Status'] == 'Unknown': res = self.__useOldPolicyRes(name = name, policyName = pName) if res['Status'] == 'NeedConfirmation': pName = p['ConfirmationPolicy'] triggeredPolicy = self.ig.C_Policies[pName] pModule = triggeredPolicy['module'] extraArgs = triggeredPolicy['args'] commandIn = triggeredPolicy['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity = granularity, name = name, status = status, policy = policy, args = args, pName = pName, pModule = pModule, extraArgs = extraArgs, commandIn = commandIn) if res['Status'] not in ('Error', 'Unknown', 'NeedConfirmation'): policyResults.append(res) return policyResults ############################################################################# def _policyCombination(self, policies): """ * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result """ if policies == []: return {} policies.sort(key=value_of_policy) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions prio, access_list, gofun = statesInfo[self.__status] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for p in policies: if value_of_policy(p) in access_list: newStatus = value_of_policy(p) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun(access_list) return { 'Status': status_of_value(newStatus), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = value_of_policy(policies[0]) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = value_of_policy(policies[0]) # At this point, a new status has been chosen. newStatus is an # integer. worstPolicies = [p for p in policies if value_of_policy(p) == newStatus] # Concatenate reasons def getReason(p): try: res = p['Reason'] except KeyError: res = '' return res worstPoliciesReasons = [getReason(p) for p in worstPolicies] def catRes(x, y): if x and y : return x + ' |###| ' + y elif x or y: if x: return x else: return y else : return '' concatenatedRes = reduce(catRes, worstPoliciesReasons, '') # Handle EndDate endDatePolicies = [p for p in worstPolicies if p.has_key('EndDate')] # Building and returning result res = {} res['Status'] = status_of_value(newStatus) if concatenatedRes != '': res['Reason'] = concatenatedRes if endDatePolicies != []: res['EndDate'] = endDatePolicies[0]['EndDate'] return res ############################################################################# def __useOldPolicyRes(self, name, policyName): """ Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} """ from DIRAC.Core.DISET.RPCClient import RPCClient rsS = RPCClient("ResourceStatus/ResourceManagement") res = rsS.getPolicyRes(name, policyName, True) if not res['OK']: raise RSSException, where(self, self.__useOldPolicyRes) + ' Could not get a policy result' res = res['Value'] if res == []: return {'Status':'Unknown'} oldStatus = res[0] oldReason = res[1] lastCheckTime = res[2] if ( lastCheckTime + datetime.timedelta(hours = 2) ) < datetime.datetime.utcnow(): return {'Status':'Unknown'} result = {} result['Status'] = oldStatus result['Reason'] = oldReason result['OLD'] = True result['PolicyName'] = policyName return result
class PDP: """ PDP = Policy Decision Point. Used to invoke policies and to take decision based on the polict results combination. """ ############################################################################# def __init__(self, VOExtension, granularity=None, name=None, status=None, formerStatus=None, reason=None, siteType=None, serviceType=None, resourceType=None, useNewRes=False): """ PDP (Policy Decision Point) initialization :params: :attr:`VOExtension`: string - VO extension (e.g. 'LHCb') :attr:`granularity`: string - a ValidRes :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.VOExtension = VOExtension self.__granularity = assignOrRaise(granularity, ValidRes, InvalidRes, self, self.__init__) self.__name = name self.__status = assignOrRaise(status, ValidStatus, InvalidStatus, self, self.__init__) self.__formerStatus = assignOrRaise(formerStatus, ValidStatus, InvalidStatus, self, self.__init__) self.__reason = reason self.__siteType = assignOrRaise(siteType, ValidSiteType, InvalidSiteType, self, self.__init__) self.__serviceType = assignOrRaise(serviceType, ValidServiceType, InvalidServiceType, self, self.__init__) self.__resourceType = assignOrRaise(resourceType, ValidResourceType, InvalidResourceType, self, self.__init__) cc = CommandCaller() self.pc = PolicyCaller(cc) self.useNewRes = useNewRes self.args = None self.policy = None self.knownInfo = None self.ig = None ############################################################################# def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ self.args = argsIn self.policy = policyIn self.knownInfo = knownInfo self.ig = InfoGetter(self.VOExtension) EVAL = self.ig.getInfoToApply(('policy', 'policyType'), granularity=self.__granularity, status=self.__status, formerStatus=self.__formerStatus, siteType=self.__siteType, serviceType=self.__serviceType, resourceType=self.__resourceType, useNewRes=self.useNewRes) for policyGroup in EVAL: policyType = policyGroup['PolicyType'] if self.policy is not None: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = self.policy.evaluate() else: if policyGroup['Policies'] is None: return { 'SinglePolicyResults': [], 'PolicyCombinedResult': { 'PolicyType': policyType, 'Action': False, 'Reason': 'No policy results' } } else: singlePolicyResults = self._invocation( self.VOExtension, self.__granularity, self.__name, self.__status, self.policy, self.args, policyGroup['Policies']) policyCombinedResults = self._policyCombination( singlePolicyResults) assert (type(policyCombinedResults) == dict) if not policyCombinedResults: return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': {} } # # policy results communication # newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy reason = policyCombinedResults['Reason'] newPolicyType = self.ig.getNewPolicyType( self.__granularity, newstatus) for npt in newPolicyType: if npt not in policyType: policyType.append(npt) decision = { 'PolicyType': policyType, 'Action': True, 'Status': newstatus, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] else: # Policies does not satisfy reason = policyCombinedResults['Reason'] decision = { 'PolicyType': policyType, 'Action': False, 'Reason': reason } if policyCombinedResults.has_key('EndDate'): decision['EndDate'] = policyCombinedResults['EndDate'] res = { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': decision } assert (type(res) == dict) return res ############################################################################# def _invocation(self, VOExtension, granularity, name, status, policy, args, policies): """ One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. """ policyResults = [] for p in policies: pName = p['Name'] pModule = p['Module'] extraArgs = p['args'] commandIn = p['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity=granularity, name=name, status=status, policy=policy, args=args, pName=pName, pModule=pModule, extraArgs=extraArgs, commandIn=commandIn) # If res is empty, return immediately if not res: return policyResults if not res.has_key('Status'): print("\n\n Policy result " + str(res) + " does not return 'Status'\n\n") raise TypeError # Else if res['Status'] == 'Unknown': res = self.__useOldPolicyRes(name=name, policyName=pName) if res['Status'] == 'NeedConfirmation': pName = p['ConfirmationPolicy'] triggeredPolicy = self.ig.C_Policies[pName] pModule = triggeredPolicy['module'] extraArgs = triggeredPolicy['args'] commandIn = triggeredPolicy['commandIn'] res = self.pc.policyInvocation(VOExtension, granularity=granularity, name=name, status=status, policy=policy, args=args, pName=pName, pModule=pModule, extraArgs=extraArgs, commandIn=commandIn) if res['Status'] not in ('Error', 'Unknown', 'NeedConfirmation'): policyResults.append(res) return policyResults ############################################################################# def _policyCombination(self, pol_results): """ INPUT: list type OUTPUT: dict type * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result """ if pol_results == []: return {} pol_results.sort(key=Status.value_of_policy) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions _prio, access_list, gofun = Status.statesInfo[self.__status] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for p in pol_results: if Status.value_of_policy(p) in access_list: newStatus = Status.value_of_policy(p) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun(access_list) return { 'Status': Status.status_of_value(newStatus), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = Status.value_of_policy(pol_results[0]) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = Status.value_of_policy(pol_results[0]) # At this point, a new status has been chosen. newStatus is an # integer. worstResults = [ p for p in pol_results if Status.value_of_policy(p) == newStatus ] # Concatenate reasons def getReason(p): try: res = p['Reason'] except KeyError: res = '' return res worstResultsReasons = [getReason(p) for p in worstResults] def catRes(x, y): if x and y: return x + ' |###| ' + y elif x or y: if x: return x else: return y else: return '' concatenatedRes = reduce(catRes, worstResultsReasons, '') # Handle EndDate endDatePolicies = [p for p in worstResults if p.has_key('EndDate')] # Building and returning result res = {} res['Status'] = Status.status_of_value(newStatus) if concatenatedRes != '': res['Reason'] = concatenatedRes if endDatePolicies != []: res['EndDate'] = endDatePolicies[0]['EndDate'] return res ############################################################################# def __useOldPolicyRes(self, name, policyName): """ Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} """ from DIRAC.Core.DISET.RPCClient import RPCClient rsS = RPCClient("ResourceStatus/ResourceManagement") res = rsS.getPolicyRes(name, policyName, True) if not res['OK']: raise RSSException, where( self, self.__useOldPolicyRes) + ' Could not get a policy result' res = res['Value'] if res == []: return {'Status': 'Unknown'} oldStatus = res[0] oldReason = res[1] lastCheckTime = res[2] if (lastCheckTime + datetime.timedelta(hours=2)) < datetime.datetime.utcnow(): return {'Status': 'Unknown'} result = {} result['Status'] = oldStatus result['Reason'] = oldReason result['OLD'] = True result['PolicyName'] = policyName return result
def testGetInfoToApply(self): ig = InfoGetter('LHCb') for g in ValidRes: for s in ValidStatus: for site_t in ValidSiteType: for service_t in ValidServiceType: if g in ('Site', 'Sites'): panel = 'Site_Panel' if g in ('Service', 'Services'): if service_t == 'Storage': panel = 'Service_Storage_Panel' if service_t == 'Computing': panel = 'Service_Computing_Panel' if service_t == 'VO-BOX': panel = 'Service_VO-BOX_Panel' if service_t == 'VOMS': panel = 'Service_VOMS_Panel' if g in ('Resource', 'Resources'): panel = 'Resource_Panel' if g in ('StorageElementRead', 'StorageElementsRead'): panel = 'SE_Panel' if g in ('StorageElementWrite', 'StorageElementsWrite'): panel = 'SE_Panel' for resource_t in ValidResourceType: res = ig.getInfoToApply(('policyType', ), g, s, None, site_t, service_t, resource_t) for p_res in res[0]['PolicyType']: self.assert_(p_res in ['Resource_PolType', 'Alarm_PolType', 'Alarm_PolType_SE']) for useNewRes in (False, True): res = ig.getInfoToApply(('policy', ), g, s, None, site_t, service_t, resource_t, useNewRes) pModuleList = [None] for k in self.configModule.Policies.keys(): try: if self.configModule.Policies[k]['module'] not in pModuleList: pModuleList.append(self.configModule.Policies[k]['module']) except KeyError: pass for p_res in res[0]['Policies']: self.assert_(p_res['Name'] in self.configModule.Policies.keys()) self.assert_(p_res['Module'] in pModuleList) if useNewRes is False: self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandIn']) self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['args']) else: try: self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandInNewRes']) except KeyError: self.assertEqual(p_res['commandIn'], self.configModule.Policies[p_res['Name']]['commandIn']) try: self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['argsNewRes']) except KeyError: self.assertEqual(p_res['args'], self.configModule.Policies[p_res['Name']]['args']) res = ig.getInfoToApply(('panel_info', ), g, s, None, site_t, service_t, resource_t, useNewRes) for p_res in res[0]['Info']: # if 'JobsEfficiencySimple' in p_res.keys(): # print useNewRes, p_res for p_name in p_res.keys(): self.assert_(p_name in self.configModule.Policies.keys()) if isinstance(p_res[p_name], list): for i in range(len(p_res[p_name])): for k in p_res[p_name][i].keys(): if useNewRes: try: self.assertEqual(p_res[p_name][i][k]['CommandIn'], self.configModule.Policies[p_name][panel][i][k]['CommandInNewRes']) except KeyError: self.assertEqual(p_res[p_name][i][k]['CommandIn'], self.configModule.Policies[p_name][panel][i][k]['CommandIn']) except TypeError: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) try: self.assertEqual(p_res[p_name][i][k]['args'], self.configModule.Policies[p_name][panel][i][k]['argsNewRes']) except KeyError: self.assertEqual(p_res[p_name][i][k]['args'], self.configModule.Policies[p_name][panel][i][k]['args']) except TypeError: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) else: try: self.assertEqual(p_res[p_name][i][k]['CommandIn'], self.configModule.Policies[p_name][panel][i][k]['CommandIn']) except: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) try: self.assertEqual(p_res[p_name][i][k]['args'], self.configModule.Policies[p_name][panel][i][k]['args']) except: self.assertEqual(p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k]) else: self.assertEqual(p_res[p_name], self.configModule.Policies[p_name][panel])
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None, WMSAdminIn = None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool( 2, 5 ) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes = False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, Utils.where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'], resourceName = name)[0][0] except IndexError: return "%s does not exist!" %name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'], storageElementName = name)[0][0] except IndexError: return "%s does not exist!" %name paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description'] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status, formerStatus = formerStatus, siteType = siteType, serviceType = serviceType, resourceType = resourceType, useNewRes = useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt(self.getInfoForPanel, args = (info, granularityForPanel, nameForPanel) ) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]} else: pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format_ = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what) info_res[format_] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, Utils.where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = { name : { 'RSSStatus': RSSStatus} } return status ############################################################################# def _getInfo(self, granularity, name, format_, what): if format_ == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName, serviceName = serviceName, serviceType = serviceType, resourceName = resourceName, storageElementName = storageElementName, gridSiteName = gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList(granularity, siteName = siteName, serviceName = serviceName, resourceName = resourceName, storageElementName = storageElementName) if res == []: return False else: return True
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__( self, **clients ): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller( cc, **clients ) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None def setup( self, granularity = None, name = None, statusType = None, status = None, formerStatus = None, reason = None, siteType = None, serviceType = None, resourceType = None, useNewRes = False ): """ PDP (Policy Decision Point) initialization :params: :attr:`granularity`: string - a ValidElement :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.__granularity = granularity self.__name = name self.__statusType = statusType self.__status = status self.__formerStatus = formerStatus self.__reason = reason self.__siteType = siteType self.__serviceType = serviceType self.__resourceType = resourceType self.__useNewRes = useNewRes ################################################################################ def takeDecision( self, policyIn = None, argsIn = None, knownInfo = None ): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ polToEval = self.iGetter.getInfoToApply( ( 'policy', 'policyType' ), granularity = self.__granularity, statusType = self.__statusType, status = self.__status, formerStatus = self.__formerStatus, siteType = self.__siteType, serviceType = self.__serviceType, resourceType = self.__resourceType, useNewRes = self.__useNewRes ) policyType = polToEval[ 'PolicyType' ] # type: generator if policyIn: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = policyIn.evaluate() else: singlePolicyResults = self._invocation( self.__granularity, self.__name, self.__status, policyIn, argsIn, polToEval['Policies'] ) policyCombinedResults = self._policyCombination( singlePolicyResults ) if policyCombinedResults == {}: policyCombinedResults[ 'Action' ] = False policyCombinedResults[ 'Reason' ] = 'No policy results' policyCombinedResults[ 'PolicyType' ] = policyType if policyCombinedResults.has_key( 'Status' ): newstatus = policyCombinedResults[ 'Status' ] if newstatus != self.__status: # Policies satisfy newPolicyType = self.iGetter.getNewPolicyType( self.__granularity, newstatus ) policyType = set( policyType ) & set( newPolicyType ) policyCombinedResults[ 'Action' ] = True else: # Policies does not satisfy policyCombinedResults[ 'Action' ] = False policyCombinedResults[ 'PolicyType' ] = policyType return { 'SinglePolicyResults' : singlePolicyResults, 'PolicyCombinedResult' : policyCombinedResults } ################################################################################ def _invocation( self, granularity, name, status, policy, args, policies ): ''' One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. Always returns a list, possibly empty. ''' policyResults = [] for pol in policies: pName = pol[ 'Name' ] pModule = pol[ 'Module' ] extraArgs = pol[ 'args' ] commandIn = pol[ 'commandIn' ] res = self.pCaller.policyInvocation( granularity = granularity, name = name, status = status, policy = policy, args = args, pName = pName, pModule = pModule, extraArgs = extraArgs, commandIn = commandIn ) # If res is empty, return immediately if not res: return policyResults if not res.has_key( 'Status' ): print('\n\n Policy result ' + str(res) + ' does not return "Status"\n\n') raise TypeError # Else if res[ 'Status' ] == 'Unknown': res = self.__useOldPolicyRes( name = name, policyName = pName ) if res[ 'Status' ] not in ( 'Error', 'Unknown' ): policyResults.append( res ) else: gLogger.warn( res ) return policyResults ################################################################################ def _policyCombination( self, pol_results ): ''' INPUT: list type OUTPUT: dict type * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result ''' if pol_results == []: return {} pol_results.sort( key=Status.value_of_policy ) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions _prio, access_list, gofun = Status.statesInfo[ self.__status ] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for polRes in pol_results: if Status.value_of_policy( polRes ) in access_list: newStatus = Status.value_of_policy( polRes ) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun( access_list ) return { 'Status': Status.status_of_value( newStatus ), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = Status.value_of_policy( pol_results[ 0 ] ) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = Status.value_of_policy( pol_results[ 0 ] ) # At this point, a new status has been chosen. newStatus is an # integer. worstResults = [ p for p in pol_results if Status.value_of_policy( p ) == newStatus ] # Concatenate reasons def getReason( pol ): try: res = pol[ 'Reason' ] except KeyError: res = '' return res worstResultsReasons = [ getReason( p ) for p in worstResults ] def catRes( xVal, yVal ): ''' Concatenate xVal and yVal. ''' if xVal and yVal : return xVal + ' |###| ' + yVal elif xVal or yVal: if xVal: return xVal else: return yVal else: return '' concatenatedRes = reduce( catRes, worstResultsReasons, '' ) # Handle EndDate endDatePolicies = [ p for p in worstResults if p.has_key( 'EndDate' ) ] # Building and returning result res = {} res[ 'Status' ] = Status.status_of_value( newStatus ) if concatenatedRes != '': res[ 'Reason' ] = concatenatedRes if endDatePolicies != []: res[ 'EndDate' ] = endDatePolicies[ 0 ][ 'EndDate' ] return res ################################################################################ def __useOldPolicyRes( self, name, policyName ): ''' Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} ''' res = self.clients[ 'ResourceManagementClient' ].getPolicyResult( name = name, policyName = policyName ) if not res[ 'OK' ]: return { 'Status' : 'Unknown' } res = res[ 'Value' ] if res == []: return { 'Status' : 'Unknown' } res = res[ 0 ] oldStatus = res[ 5 ] oldReason = res[ 6 ] lastCheckTime = res[ 8 ] if ( lastCheckTime + datetime.timedelta(hours = 2) ) < datetime.datetime.utcnow(): return { 'Status' : 'Unknown' } result = {} result[ 'Status' ] = oldStatus result[ 'Reason' ] = oldReason result[ 'OLD' ] = True result[ 'PolicyName' ] = policyName return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*']) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes=False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList( 'Resource', ['ResourceType'], resourceName=name)[0][0] except IndexError: return "%s does not exist!" % name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList( 'StorageElement', ['SiteType'], storageElementName=name)[0][0] except IndexError: return "%s does not exist!" % name paramNames = [ 'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description' ] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status=status, formerStatus=formerStatus, siteType=siteType, serviceType=serviceType, resourceType=resourceType, useNewRes=useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel][ 'DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel][ 'RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt( self.getInfoForPanel, args=(info, granularityForPanel, nameForPanel)) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy][ 'Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]} else: pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format, what) info_res[format] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name: { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = {name: {'RSSStatus': RSSStatus}} return status ############################################################################# def _getInfo(self, granularity, name, format, what): if format == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList( gran, paramsList=paramsL, siteName=siteName, serviceName=serviceName, serviceType=serviceType, resourceName=resourceName, storageElementName=storageElementName, gridSiteName=gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList( granularity, siteName=siteName, serviceName=serviceName, resourceName=resourceName, storageElementName=storageElementName) if res == []: return False else: return True
class PDP: """ The PDP (Policy Decision Point) module is used to: 1. Decides which policies have to be applied. 2. Invokes an evaluation of the policies, and returns the result (to a PEP) """ def __init__(self, **clients): ''' Constructor. Defines members that will be used later on. ''' cc = CommandCaller() self.clients = clients self.pCaller = PolicyCaller(cc, **clients) self.iGetter = InfoGetter() self.__granularity = None self.__name = None self.__statusType = None self.__status = None self.__formerStatus = None self.__reason = None self.__siteType = None self.__serviceType = None self.__resourceType = None self.__useNewRes = None def setup(self, granularity=None, name=None, statusType=None, status=None, formerStatus=None, reason=None, siteType=None, serviceType=None, resourceType=None, useNewRes=False): """ PDP (Policy Decision Point) initialization :params: :attr:`granularity`: string - a ValidElement :attr:`name`: string - name (e.g. of a site) :attr:`status`: string - status :attr:`formerStatus`: string - former status :attr:`reason`: string - optional reason for last status change :attr:`siteType`: string - optional site type :attr:`serviceType`: string - optional service type :attr:`resourceType`: string - optional resource type """ self.__granularity = granularity self.__name = name self.__statusType = statusType self.__status = status self.__formerStatus = formerStatus self.__reason = reason self.__siteType = siteType self.__serviceType = serviceType self.__resourceType = resourceType self.__useNewRes = useNewRes ################################################################################ def takeDecision(self, policyIn=None, argsIn=None, knownInfo=None): """ PDP MAIN FUNCTION decides policies that have to be applied, based on __granularity, __name, __status, __formerStatus __reason If more than one policy is evaluated, results are combined. Logic for combination: a conservative approach is followed (i.e. if a site should be banned for at least one policy, that's what is returned) returns: { 'PolicyType': a policyType (in a string), 'Action': True|False, 'Status': 'Active'|'Probing'|'Banned', 'Reason': a reason 'EndDate: datetime.datetime (in a string)} """ polToEval = self.iGetter.getInfoToApply( ('policy', 'policyType'), granularity=self.__granularity, statusType=self.__statusType, status=self.__status, formerStatus=self.__formerStatus, siteType=self.__siteType, serviceType=self.__serviceType, resourceType=self.__resourceType, useNewRes=self.__useNewRes) policyType = polToEval['PolicyType'] # type: generator if policyIn: # Only the policy provided will be evaluated # FIXME: Check that the policies are valid. singlePolicyResults = policyIn.evaluate() else: singlePolicyResults = self._invocation(self.__granularity, self.__name, self.__status, policyIn, argsIn, polToEval['Policies']) policyCombinedResults = self._policyCombination(singlePolicyResults) if policyCombinedResults == {}: policyCombinedResults['Action'] = False policyCombinedResults['Reason'] = 'No policy results' policyCombinedResults['PolicyType'] = policyType if policyCombinedResults.has_key('Status'): newstatus = policyCombinedResults['Status'] if newstatus != self.__status: # Policies satisfy newPolicyType = self.iGetter.getNewPolicyType( self.__granularity, newstatus) policyType = set(policyType) & set(newPolicyType) policyCombinedResults['Action'] = True else: # Policies does not satisfy policyCombinedResults['Action'] = False policyCombinedResults['PolicyType'] = policyType return { 'SinglePolicyResults': singlePolicyResults, 'PolicyCombinedResult': policyCombinedResults } ################################################################################ def _invocation(self, granularity, name, status, policy, args, policies): ''' One by one, use the PolicyCaller to invoke the policies, and putting their results in `policyResults`. When the status is `Unknown`, invokes `self.__useOldPolicyRes`. Always returns a list, possibly empty. ''' policyResults = [] for pol in policies: pName = pol['Name'] pModule = pol['Module'] extraArgs = pol['args'] commandIn = pol['commandIn'] res = self.pCaller.policyInvocation(granularity=granularity, name=name, status=status, policy=policy, args=args, pName=pName, pModule=pModule, extraArgs=extraArgs, commandIn=commandIn) # If res is empty, return immediately if not res: return policyResults if not res.has_key('Status'): print('\n\n Policy result ' + str(res) + ' does not return "Status"\n\n') raise TypeError # Else if res['Status'] == 'Unknown': res = self.__useOldPolicyRes(name=name, policyName=pName) if res['Status'] not in ('Error', 'Unknown'): policyResults.append(res) else: gLogger.warn(res) return policyResults ################################################################################ def _policyCombination(self, pol_results): ''' INPUT: list type OUTPUT: dict type * Compute a new status, and store it in variable newStatus, of type integer. * Make a list of policies that have the worst result. * Concatenate the Reason fields * Take the first EndDate field that exists (FIXME: Do something more clever) * Finally, return the result ''' if pol_results == []: return {} pol_results.sort(key=Status.value_of_policy) newStatus = -1 # First, set an always invalid status try: # We are in a special status, maybe forbidden transitions _prio, access_list, gofun = Status.statesInfo[self.__status] if access_list != set(): # Restrictions on transitions, checking if one is suitable: for polRes in pol_results: if Status.value_of_policy(polRes) in access_list: newStatus = Status.value_of_policy(polRes) break # No status from policies suitable, applying stategy and # returning result. if newStatus == -1: newStatus = gofun(access_list) return { 'Status': Status.status_of_value(newStatus), 'Reason': 'Status forced by PDP' } else: # Special Status, but no restriction on transitions newStatus = Status.value_of_policy(pol_results[0]) except KeyError: # We are in a "normal" status: All transitions are possible. newStatus = Status.value_of_policy(pol_results[0]) # At this point, a new status has been chosen. newStatus is an # integer. worstResults = [ p for p in pol_results if Status.value_of_policy(p) == newStatus ] # Concatenate reasons def getReason(pol): try: res = pol['Reason'] except KeyError: res = '' return res worstResultsReasons = [getReason(p) for p in worstResults] def catRes(xVal, yVal): ''' Concatenate xVal and yVal. ''' if xVal and yVal: return xVal + ' |###| ' + yVal elif xVal or yVal: if xVal: return xVal else: return yVal else: return '' concatenatedRes = reduce(catRes, worstResultsReasons, '') # Handle EndDate endDatePolicies = [p for p in worstResults if p.has_key('EndDate')] # Building and returning result res = {} res['Status'] = Status.status_of_value(newStatus) if concatenatedRes != '': res['Reason'] = concatenatedRes if endDatePolicies != []: res['EndDate'] = endDatePolicies[0]['EndDate'] return res ################################################################################ def __useOldPolicyRes(self, name, policyName): ''' Use the RSS Service to get an old policy result. If such result is older than 2 hours, it returns {'Status':'Unknown'} ''' res = self.clients['ResourceManagementClient'].getPolicyResult( name=name, policyName=policyName) if not res['OK']: return {'Status': 'Unknown'} res = res['Value'] if res == []: return {'Status': 'Unknown'} res = res[0] oldStatus = res[5] oldReason = res[6] lastCheckTime = res[8] if (lastCheckTime + datetime.timedelta(hours=2)) < datetime.datetime.utcnow(): return {'Status': 'Unknown'} result = {} result['Status'] = oldStatus result['Reason'] = oldReason result['OLD'] = True result['PolicyName'] = policyName return result ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def testGetInfoToApply(self): ig = InfoGetter("LHCb") for g in ValidRes: for s in ValidStatus: for site_t in ValidSiteType: for service_t in ValidServiceType: if g in ("Site", "Sites"): panel = "Site_Panel" if g in ("Service", "Services"): if service_t == "Storage": panel = "Service_Storage_Panel" if service_t == "Computing": panel = "Service_Computing_Panel" if service_t == "VO-BOX": panel = "Service_VO-BOX_Panel" if service_t == "VOMS": panel = "Service_VOMS_Panel" if g in ("Resource", "Resources"): panel = "Resource_Panel" if g in ("StorageElementRead", "StorageElementsRead"): panel = "SE_Panel" if g in ("StorageElementWrite", "StorageElementsWrite"): panel = "SE_Panel" for resource_t in ValidResourceType: ## Testing the policyType (__getPolTypes) part res = ig.getInfoToApply(("policyType",), g, None, s, None, site_t, service_t, resource_t) for p_res in res["PolicyType"]: self.assert_(p_res in CS.getTypedDictRootedAt("PolicyTypes").keys()) for useNewRes in (False, True): ## Testing the policy (__getPolToEval) part res = ig.getInfoToApply( ("policy",), g, None, s, None, site_t, service_t, resource_t, useNewRes ) pModuleList = [] for k in self.configModule.Policies.keys(): try: if self.configModule.Policies[k]["module"] not in pModuleList: pModuleList.append(self.configModule.Policies[k]["module"]) except KeyError: pass for p_res in res["Policies"]: # All __getPolToEval results... self.assertTrue(p_res["Name"] in CS.getTypedDictRootedAt("Policies")) # self.assertTrue(p_res['Module'] in pModuleList) if useNewRes is False: self.assertEqual( p_res["commandIn"], self.configModule.Policies[p_res["Name"]]["commandIn"] ) self.assertEqual( p_res["args"], self.configModule.Policies[p_res["Name"]]["args"] ) else: try: self.assertEqual( p_res["commandIn"], self.configModule.Policies[p_res["Name"]]["commandInNewRes"], ) except KeyError: self.assertEqual( p_res["commandIn"], self.configModule.Policies[p_res["Name"]]["commandIn"], ) try: self.assertEqual( p_res["args"], self.configModule.Policies[p_res["Name"]]["argsNewRes"] ) except KeyError: self.assertEqual( p_res["args"], self.configModule.Policies[p_res["Name"]]["args"] ) res = ig.getInfoToApply( ("panel_info",), g, None, s, None, site_t, service_t, resource_t, useNewRes ) for p_res in res["Info"]: # if 'JobsEfficiencySimple' in p_res.keys(): # print useNewRes, p_res for p_name in p_res.keys(): self.assert_(p_name in self.configModule.Policies.keys()) if isinstance(p_res[p_name], list): for i in range(len(p_res[p_name])): for k in p_res[p_name][i].keys(): if useNewRes: try: self.assertEqual( p_res[p_name][i][k]["CommandIn"], self.configModule.Policies[p_name][panel][i][k][ "CommandInNewRes" ], ) except KeyError: self.assertEqual( p_res[p_name][i][k]["CommandIn"], self.configModule.Policies[p_name][panel][i][k][ "CommandIn" ], ) except TypeError: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) try: self.assertEqual( p_res[p_name][i][k]["args"], self.configModule.Policies[p_name][panel][i][k][ "argsNewRes" ], ) except KeyError: self.assertEqual( p_res[p_name][i][k]["args"], self.configModule.Policies[p_name][panel][i][k]["args"], ) except TypeError: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) else: try: self.assertEqual( p_res[p_name][i][k]["CommandIn"], self.configModule.Policies[p_name][panel][i][k][ "CommandIn" ], ) except: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) try: self.assertEqual( p_res[p_name][i][k]["args"], self.configModule.Policies[p_name][panel][i][k]["args"], ) except: self.assertEqual( p_res[p_name][i][k], self.configModule.Policies[p_name][panel][i][k], ) else: self.assertEqual(p_res[p_name], self.configModule.Policies[p_name][panel])