def __init__(self, clients=None):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
if clients is None:
clients = {}
# PEP uses internally two of the clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' in clients:
self.rsClient = clients['ResourceStatusClient']
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(clients)
def __init__(self, clients=dict()):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
self.clients = dict(clients)
# Creating the client in the PEP is a convenience for the PDP, that
# uses internally the two TSS clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' not in clients:
self.clients['ResourceStatusClient'] = ResourceStatusClient()
if 'ResourceManagementClient' not in clients:
self.clients[
'ResourceManagementClient'] = ResourceManagementClient()
if 'SiteStatus' not in clients:
self.clients['SiteStatus'] = SiteStatus()
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(self.clients)
self.log = gLogger
def __init__(self, clients=dict()):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
self.clients = dict(clients)
# Creating the client in the PEP is a convenience for the PDP, that uses internally the RSS clients
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' %
res['Message'])
raise ImportError(res['Message'])
rsClass = res['Value']
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error(
'Failed to load ResourceManagementClient class: %s' %
res['Message'])
raise ImportError(res['Message'])
rmClass = res['Value']
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.SiteStatus', 'SiteStatus')
if not res['OK']:
self.log.error('Failed to load SiteStatus class: %s' %
res['Message'])
raise ImportError(res['Message'])
ssClass = res['Value']
if 'ResourceStatusClient' not in clients:
self.clients['ResourceStatusClient'] = rsClass()
if 'ResourceManagementClient' not in clients:
self.clients['ResourceManagementClient'] = rmClass()
if 'SiteStatus' not in clients:
self.clients['SiteStatus'] = ssClass()
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(self.clients)
self.log = gLogger
def test__policyCombination(self):
for granularity in ValidRes:
for status in ValidStatus:
for oldStatus in ValidStatus:
if status == oldStatus:
continue
for newStatus1 in ValidStatus:
for newStatus2 in ValidStatus:
pdp = PDP(self.VO, granularity, 'XX', status, oldStatus, 'XX')
polRes = {'Status':newStatus1, 'Reason':'-Reason1-'}
polRes2 = {'Status':newStatus2, 'Reason':'-Reason2-'}
# 0 policies
res = pdp._policyCombination([])
self.assertEqual(res, {})
# 1 policy
res = pdp._policyCombination([polRes])
if status == 'Banned':
self.assertTrue(value_of_status(res['Status']) <= 1)
if status == 'Banned' and newStatus1 in ['Active','Bad','Probing']:
self.assertEqual(res['Status'], 'Probing')
else:
self.assertEqual(res['Status'], newStatus1)
# 2 policies
res = pdp._policyCombination([polRes, polRes2])
if status == 'Banned':
self.assertTrue(value_of_status(res['Status']) <= 1)
if status == 'Banned' and newStatus1 in ['Active','Bad','Probing'] and newStatus2 in ['Active','Bad','Probing']:
self.assertEqual(res['Status'], 'Probing')
if status != 'Banned' and value_of_status(newStatus1) < value_of_status(newStatus1):
self.assertEqual(res['Status'], newStatus1)
if status != 'Banned' and value_of_status(newStatus2) < value_of_status(newStatus1):
self.assertEqual(res['Status'], newStatus2)
# all different policies
def make_polres(status):
return { 'Status': status, 'Reason': 'Because of ' + status }
all_polres = map(make_polres, ValidStatus)
res = pdp._policyCombination(all_polres)
self.assertEqual(res['Status'], 'Banned')
def __init__( self, clients = dict() ):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
self.clients = dict( clients )
# Creating the client in the PEP is a convenience for the PDP, that
# uses internally the two TSS clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' not in clients:
self.clients['ResourceStatusClient'] = ResourceStatusClient()
if 'ResourceManagementClient' not in clients:
self.clients['ResourceManagementClient'] = ResourceManagementClient()
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP( self.clients )
self.log = gLogger
def __init__(self, clients=None):
if clients is None:
clients = {}
if 'ResourceStatusClient' in clients:
self.rsClient = clients['ResourceStatusClient']
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
self.pdp = PDP(clients)
def __init__( self, clients = None ):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
if clients is None:
clients = {}
# PEP uses internally two of the clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' in clients:
self.rsClient = clients[ 'ResourceStatusClient' ]
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP( clients )
def test_PolicyFail(self):
self.mock_p.evaluate.side_effect = RSSException()
for granularity in ValidRes:
for status in ValidStatus:
for oldStatus in ValidStatus:
if status == oldStatus:
continue
pdp = PDP(self.VO, granularity, 'XX', status, oldStatus, 'XX')
self.failUnlessRaises(Exception, pdp.takeDecision, self.mock_pdp, self.mock_rsDB)
self.failUnlessRaises(Exception, pdp.takeDecision, self.mock_pdp, self.mock_rsDB, knownInfo={'DT':'AT_RISK'})
def test_takeDecision(self):
for granularity in ValidRes:
for status in ValidStatus:
for oldStatus in ValidStatus:
if status == oldStatus: continue
self.mock_p.evaluate.return_value = [{'Status': status, 'Reason': 'testReason', 'PolicyName': 'test_P'}]
pdp = PDP(self.VO, granularity, 'XX', oldStatus, None, 'XX')
res = pdp.takeDecision(policyIn = self.mock_p)
res = res['PolicyCombinedResult']
self.assert_(res['Action'])
res = pdp.takeDecision(policyIn = self.mock_p, argsIn = ())
res = res['PolicyCombinedResult']
self.assert_(res['Action'])
res = pdp.takeDecision(policyIn = self.mock_p, knownInfo={})
res = res['PolicyCombinedResult']
self.assert_(res['Action'])
def test_takeDecision(self):
for granularity in ValidRes:
for status in ValidStatus:
for oldStatus in ValidStatus:
if status == oldStatus: continue
self.mock_p.evaluate.return_value = [{'Status': status, 'Reason': 'testReason', 'PolicyName': 'test_P'}]
pdp = PDP(self.VO, granularity, 'XX', oldStatus, None, 'XX')
res = pdp.takeDecision(policyIn = self.mock_p)
res = res['PolicyCombinedResult']
for r in res:
self.assert_(r['Action'])
res = pdp.takeDecision(policyIn = self.mock_p, argsIn = ())
res = res['PolicyCombinedResult']
for r in res:
self.assert_(r['Action'])
res = pdp.takeDecision(policyIn = self.mock_p, knownInfo={})
res = res['PolicyCombinedResult']
for r in res:
self.assert_(r['Action'])
def __init__(self, pdp=None, clients=None):
'''
Enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdp` : a custom PDP object (optional)
:attr:`clients` : a dictionary containing modules corresponding to clients.
'''
if clients is None:
clients = {}
try:
self.rsClient = clients['ResourceStatusClient']
except KeyError:
self.rsClient = ResourceStatusClient()
try:
self.rmClient = clients['ResourceManagementClient']
except KeyError:
self.rmClient = ResourceManagementClient()
self.clients = clients
if not pdp:
self.pdp = PDP(**clients)
def __init__( self, pdp = None, clients = None ):
'''
Enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdp` : a custom PDP object (optional)
:attr:`clients` : a dictionary containing modules corresponding to clients.
'''
if clients is None:
clients = {}
try:
self.rsClient = clients[ 'ResourceStatusClient' ]
except KeyError:
self.rsClient = ResourceStatusClient()
try:
self.rmClient = clients[ 'ResourceManagementClient' ]
except KeyError:
self.rmClient = ResourceManagementClient()
self.clients = clients
if not pdp:
self.pdp = PDP( **clients )
class PEP:
def __init__(self, clients=None):
if clients is None:
clients = {}
if 'ResourceStatusClient' in clients:
self.rsClient = clients['ResourceStatusClient']
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
self.pdp = PDP(clients)
def enforce(self, decissionParams):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## policy decision point setup #############################################
self.pdp.setup(decissionParams)
## policy decision #########################################################
resDecisions = self.pdp.takeDecision()
if not resDecisions['OK']:
gLogger.error(
'PEP: Something went wrong, not enforcing policies for %s' %
decissionParams)
return resDecisions
resDecisions = resDecisions['Value']
# We take from PDP the decision parameters used to find the policies
decissionParams = resDecisions['decissionParams']
policyCombinedResult = resDecisions['policyCombinedResult']
singlePolicyResults = resDecisions['singlePolicyResults']
for policyActionName, policyActionType in policyCombinedResult[
'PolicyAction']:
try:
actionMod = Utils.voimport(
'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' %
policyActionType)
except ImportError:
gLogger.error('Error importing %s action' % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
gLogger.error('Error importing %s action class' %
policyActionType)
continue
actionObj = action(policyActionName, decissionParams,
policyCombinedResult, singlePolicyResults,
self.clients)
gLogger.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult['OK']:
gLogger.error(actionResult['Message'])
return S_OK(resDecisions)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class PEP(object):
"""PEP ( Policy Enforcement Point )"""
def __init__(self, clients=dict()):
"""Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
self.clients = dict(clients)
# Creating the client in the PEP is a convenience for the PDP, that uses internally the RSS clients
res = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceStatusClient")
if not res["OK"]:
self.log.error("Failed to load ResourceStatusClient class: %s" % res["Message"])
raise ImportError(res["Message"])
rsClass = res["Value"]
res = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient")
if not res["OK"]:
self.log.error("Failed to load ResourceManagementClient class: %s" % res["Message"])
raise ImportError(res["Message"])
rmClass = res["Value"]
res = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.SiteStatus")
if not res["OK"]:
self.log.error("Failed to load SiteStatus class: %s" % res["Message"])
raise ImportError(res["Message"])
ssClass = res["Value"]
if "ResourceStatusClient" not in clients:
self.clients["ResourceStatusClient"] = rsClass()
if "ResourceManagementClient" not in clients:
self.clients["ResourceManagementClient"] = rmClass()
if "SiteStatus" not in clients:
self.clients["SiteStatus"] = ssClass()
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(self.clients)
self.log = gLogger
def enforce(self, decisionParams):
"""Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
if not decisionParams:
self.log.warn("No decision params...?")
return S_OK()
standardParamsDict = {
"element": None,
"name": None,
"elementType": None,
"statusType": None,
"status": None,
"reason": None,
"tokenOwner": None,
# Last parameter allows policies to be de-activated
"active": "Active",
}
standardParamsDict.update(decisionParams)
if standardParamsDict["element"] is not None:
self.log = gLogger.getSubLogger("PEP/%s" % standardParamsDict["element"])
if standardParamsDict["name"] is not None:
self.log = gLogger.getSubLogger(
"PEP/%s/%s" % (standardParamsDict["element"], standardParamsDict["name"])
)
self.log.verbose(
"Enforce - statusType: %s, status: %s"
% (standardParamsDict["statusType"], standardParamsDict["status"])
)
decisionParams = dict(standardParamsDict)
# Setup PDP with new parameters dictionary
self.pdp.setup(decisionParams)
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions["OK"]:
self.log.error("Something went wrong, not enforcing policies", "%s" % decisionParams)
return resDecisions
resDecisions = resDecisions["Value"]
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions["decisionParams"]
policyCombinedResult = resDecisions["policyCombinedResult"]
singlePolicyResults = resDecisions["singlePolicyResults"]
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated(decisionParams)
if not isNotUpdated["OK"]:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult["PolicyAction"]:
try:
actionMod = Utils.voimport("DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s" % policyActionType)
except ImportError:
self.log.error("Error importing %s action" % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
self.log.error("Error importing %s action class" % policyActionType)
continue
actionObj = action(
policyActionName, decisionParams, policyCombinedResult, singlePolicyResults, self.clients
)
self.log.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult["OK"]:
self.log.error(actionResult["Message"])
return S_OK(resDecisions)
def __isNotUpdated(self, decisionParams):
"""Checks for the existence of the element as it was passed to the PEP. It may
happen that while being the element processed by the PEP an user through the
web interface or the CLI has updated the status for this particular element. As
a result, the PEP would overwrite whatever the user had set. This check is not
perfect, as still an user action can happen while executing the actions, but
the probability is close to 0. However, if there is an action that takes seconds
to be executed, this must be re-evaluated. !
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies
:return: S_OK / S_ERROR
"""
# Copy original dictionary and get rid of one key we cannot pass as kwarg
selectParams = dict(decisionParams)
del selectParams["element"]
del selectParams["active"]
# We expect to have an exact match. If not, then something has changed and
# we cannot proceed with the actions.
if decisionParams["element"] == "Site":
unchangedRow = self.clients["SiteStatus"].getSiteStatuses([decisionParams["name"]])
else:
unchangedRow = self.clients["ResourceStatusClient"].selectStatusElement(
decisionParams["element"], "Status", **selectParams
)
if not unchangedRow["OK"]:
return unchangedRow
if not unchangedRow["Value"]:
msg = "%(name)s ( %(status)s / %(statusType)s ) has been updated after PEP started running" % selectParams
self.log.error(msg)
return S_ERROR(msg)
return S_OK()
def test_site( self ):
pdp = PDP()
# empty
pdp.setup( None )
res = pdp.takeDecision()
self.assert_( res['OK'] )
# site
decisionParams = {'element' : 'Site',
'name' : 'Site1',
'elementType' : None,
'statusType' : 'ReadAccess',
'status' : 'Active',
'reason' : None,
'tokenOwner' : None}
pdp.setup( decisionParams )
res = pdp.takeDecision()
self.assert_( res['OK'] )
self.assertEqual( res['Value']['policyCombinedResult']['Status'], 'Banned' )
# site2
decisionParams = {'element' : 'Site',
'name' : 'Site2',
'elementType' : 'CE',
'statusType' : 'ReadAccess',
'status' : 'Active',
'domain' : 'test',
'reason' : None,
'tokenOwner' : None}
pdp.setup( decisionParams )
res = pdp.takeDecision()
self.assert_( res['OK'] )
self.assertEqual( res['Value']['policyCombinedResult']['Status'], 'Banned' )
# mySE
decisionParams = {'element' : 'Resource',
'name' : 'mySE',
'elementType' : 'StorageElement',
'statusType' : 'ReadAccess',
'status' : 'Active',
'reason' : None,
'tokenOwner' : None}
pdp.setup( decisionParams )
res = pdp.takeDecision()
self.assert_( res['OK'] )
self.assertEqual( res['Value']['policyCombinedResult']['Status'], 'Active' )
# SE1
decisionParams = {'element' : 'Resource',
'name' : 'SE1',
'elementType' : 'StorageElement',
'statusType' : 'ReadAccess',
'status' : 'Active',
'reason' : None,
'tokenOwner' : None}
pdp.setup( decisionParams )
res = pdp.takeDecision()
self.assert_( res['OK'] )
self.assertEqual( res['Value']['policyCombinedResult']['Status'], 'Banned' )
def test__policyCombination(self):
for granularity in ValidRes:
for status in ValidStatus:
for oldStatus in ValidStatus:
if status == oldStatus:
continue
for newStatus1 in ValidStatus:
for newStatus2 in ValidStatus:
pdp = PDP(self.VO, granularity, 'XX', status,
oldStatus, 'XX')
polRes = {
'Status': newStatus1,
'Reason': '-Reason1-'
}
polRes2 = {
'Status': newStatus2,
'Reason': '-Reason2-'
}
# 0 policies
res = pdp._policyCombination([])
self.assertEqual(res, {})
# 1 policy
res = pdp._policyCombination([polRes])
if status == 'Banned':
self.assertTrue(
value_of_status(res['Status']) <= 1)
if status == 'Banned' and newStatus1 in [
'Active', 'Bad', 'Probing'
]:
self.assertEqual(res['Status'], 'Probing')
else:
self.assertEqual(res['Status'], newStatus1)
# 2 policies
res = pdp._policyCombination([polRes, polRes2])
if status == 'Banned':
self.assertTrue(
value_of_status(res['Status']) <= 1)
if status == 'Banned' and newStatus1 in [
'Active', 'Bad', 'Probing'
] and newStatus2 in ['Active', 'Bad', 'Probing']:
self.assertEqual(res['Status'], 'Probing')
if status != 'Banned' and value_of_status(
newStatus1) < value_of_status(newStatus1):
self.assertEqual(res['Status'], newStatus1)
if status != 'Banned' and value_of_status(
newStatus2) < value_of_status(newStatus1):
self.assertEqual(res['Status'], newStatus2)
# all different policies
def make_polres(status):
return {
'Status': status,
'Reason': 'Because of ' + status
}
all_polres = map(make_polres, ValidStatus)
res = pdp._policyCombination(all_polres)
self.assertEqual(res['Status'], 'Banned')
def enforce(self, pdpIn = None, rsDBIn = None, rmDBIn = None, ncIn = None, setupIn = None,
daIn = None, csAPIIn = None, knownInfo = None):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP(self.VOExtension, granularity = self.__granularity, name = self.__name,
status = self.__status, formerStatus = self.__formerStatus, reason = self.__reason,
siteType = self.__siteType, serviceType = self.__serviceType,
resourceType = self.__resourceType, useNewRes = self.useNewRes)
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = getSetup()['Value']
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
# policy decision
resDecisions = pdp.takeDecision(knownInfo=knownInfo)
# if self.__name == 'CERN-RAW':
# print resDecisions
for res in resDecisions['PolicyCombinedResult']:
self.__policyType = res['PolicyType']
#if self.__realBan == False:
# continue
if 'Resource_PolType' in self.__policyType:
# If token != RS_SVC, we do not update the token, just the LastCheckedTime
if self.__realBan == False:
rsDB.setLastMonitoredCheckTime(self.__granularity, self.__name)
else:
self._ResourcePolTypeActions(resDecisions, res, rsDB, rmDB)
if 'Alarm_PolType' in self.__policyType:
self._AlarmPolTypeActions(res, nc, setup, rsDB)
if 'RealBan_PolType' in self.__policyType and self.__realBan == True:
self._RealBanPolTypeActions(res, da, csAPI, setup)
if 'Collective_PolType' in self.__policyType:
# do something
pass
class PEP:
""" PEP ( Policy Enforcement Point )
"""
def __init__(self, clients=None):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
if clients is None:
clients = {}
# PEP uses internally two of the clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' in clients:
self.rsClient = clients['ResourceStatusClient']
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(clients)
def enforce(self, decisionParams):
""" Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
# Setup PDP with new parameters dictionary
self.pdp.setup(decisionParams)
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions['OK']:
gLogger.error(
'PEP: Something went wrong, not enforcing policies for %s' %
decisionParams)
return resDecisions
resDecisions = resDecisions['Value']
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions['decissionParams']
policyCombinedResult = resDecisions['policyCombinedResult']
singlePolicyResults = resDecisions['singlePolicyResults']
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated(decisionParams)
if not isNotUpdated['OK']:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult[
'PolicyAction']:
try:
actionMod = Utils.voimport(
'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' %
policyActionType)
except ImportError:
gLogger.error('Error importing %s action' % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
gLogger.error('Error importing %s action class' %
policyActionType)
continue
actionObj = action(policyActionName, decisionParams,
policyCombinedResult, singlePolicyResults,
self.clients)
gLogger.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult['OK']:
gLogger.error(actionResult['Message'])
return S_OK(resDecisions)
def __isNotUpdated(self, decisionParams):
""" Checks for the existence of the element as it was passed to the PEP. It may
happen that while being the element processed by the PEP an user through the
web interface or the CLI has updated the status for this particular element. As
a result, the PEP would overwrite whatever the user had set. This check is not
perfect, as still an user action can happen while executing the actions, but
the probability is close to 0. However, if there is an action that takes seconds
to be executed, this must be re-evaluated. !
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies
:return: S_OK / S_ERROR
"""
# Copy original dictionary and get rid of one key we cannot pass as kwarg
selectParams = decisionParams.copy()
del selectParams['element']
del selectParams['active']
# We expect to have an exact match. If not, then something has changed and
# we cannot proceed with the actions.
unchangedRow = self.rsClient.selectStatusElement(
decisionParams['element'], 'Status', **selectParams)
if not unchangedRow['OK']:
return unchangedRow
if not unchangedRow['Value']:
msg = '%(name)s ( %(status)s / %(statusType)s ) has been updated after PEP started running'
return S_ERROR(msg % selectParams)
return S_OK()
#...............................................................................
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def execute( self ):
'''
The main TokenAgent execution method.
Checks for tokens owned by users that are expiring, and notifies those users.
Calls rsClient.setToken() to set 'RS_SVC' as owner for those tokens that expired.
'''
adminMail = ''
try:
reason = 'Out of date token'
#reAssign the token to RS_SVC
#for g in self.ELEMENTS:
validElements = RssConfiguration.getValidElements()
for granularity in validElements:
tokensExpired = self.rsClient.getTokens( granularity,
tokenExpiration = datetime.datetime.utcnow() )
if tokensExpired[ 'Value' ]:
adminMail += '\nLIST OF EXPIRED %s TOKENS\n' % granularity
adminMail += '%s|%s|%s\n' % ( 'user'.ljust(20), 'name'.ljust(15), 'status type')
for token in tokensExpired[ 'Value' ]:
name = token[ 1 ]
stype = token[ 2 ]
user = token[ 9 ]
self.rsClient.setToken( granularity, name, stype, reason, 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
adminMail += ' %s %s %s\n' %( user.ljust(20), name.ljust(15), stype )
#notify token owners
inNHours = datetime.datetime.utcnow() + datetime.timedelta( hours = self.notifyHours )
#for g in self.ELEMENTS:
for granularity in validElements:
tokensExpiring = self.rsClient.getTokens( granularity, tokenExpiration = inNHours )
if tokensExpiring[ 'Value' ]:
adminMail += '\nLIST OF EXPIRING %s TOKENS\n' % granularity
adminMail += '%s|%s|%s\n' % ( 'user'.ljust(20),'name'.ljust(15),'status type')
for token in tokensExpiring[ 'Value' ]:
name = token[ 1 ]
stype = token[ 2 ]
user = token[ 9 ]
adminMail += '\n %s %s %s\n' %( user.ljust(20), name.ljust(15), stype )
#If user is RS_SVC, we ignore this, whenever the token is out, this
#agent will set again the token to RS_SVC
if user == 'RS_SVC':
continue
pdp = PDP( granularity = granularity, name = name, statusType = stype )
decision = pdp.takeDecision()
pcresult = decision[ 'PolicyCombinedResult' ]
spresult = decision[ 'SinglePolicyResults' ]
expiration = token[ 10 ]
mailMessage = "The token for %s %s ( %s )" % ( granularity, name, stype )
mailMessage = mailMessage + " will expire on %s\n\n" % expiration
mailMessage = mailMessage + "You can renew it with command 'dirac-rss-renew-token'.\n"
mailMessage = mailMessage + "If you don't take any action, RSS will take control of the resource.\n\n"
policyMessage = ''
if pcresult[ 'Action' ]:
policyMessage += " Policies applied will set status to %s.\n" % pcresult[ 'Status' ]
for spr in spresult:
policyMessage += " %s Status->%s\n" % ( spr[ 'PolicyName' ].ljust(25), spr[ 'Status' ] )
mailMessage += policyMessage
adminMail += policyMessage
self.noClient.sendMail( self.rmClient.getUserRegistryCache( user )[ 'Value' ][ 0 ][ 2 ],
'Token for %s is expiring' % name, mailMessage )
if adminMail != '':
#FIXME: 'ubeda' is not generic ;p
self.noClient.sendMail( self.rmClient.getUserRegistryCache( 'ubeda' )[ 'Value' ][ 0 ][ 2 ],
"Token's summary", adminMail )
return S_OK()
except Exception:
errorStr = "TokenAgent execution"
self.log.exception( errorStr )
return S_ERROR( errorStr )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def enforce(self, pdpIn = None, rsDBIn = None, rmDBIn = None, ncIn = None, setupIn = None,
daIn = None, csAPIIn = None, knownInfo = None):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP(self.VOExtension, granularity = self.__granularity, name = self.__name,
status = self.__status, formerStatus = self.__formerStatus, reason = self.__reason,
siteType = self.__siteType, serviceType = self.__serviceType,
resourceType = self.__resourceType, useNewRes = self.useNewRes)
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = CS.getSetup()['Value']
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
###################
# policy decision #
###################
resDecisions = pdp.takeDecision(knownInfo=knownInfo)
if resDecisions != {}:
res = resDecisions['PolicyCombinedResult']
policyType = res['PolicyType']
if 'Resource_PolType' in policyType:
ResourcePolTypeActions(self.__granularity, self.__name, resDecisions, res, rsDB, rmDB)
if 'Alarm_PolType' in policyType:
AlarmPolTypeActions(self.__granularity, self.__name,
self.__siteType, self.__serviceType, self.__resourceType,
res, nc, setup, rsDB)
if 'RealBan_PolType' in policyType and self.__realBan == True:
RealBanPolTypeActions(self.__granularity, self.__name, res, da, csAPI, setup)
def test_site(self):
pdp = PDP()
# empty
pdp.setup(None)
res = pdp.takeDecision()
self.assertTrue(res["OK"])
# site
decisionParams = {
"element": "Site",
"name": "Site1",
"elementType": None,
"statusType": "ReadAccess",
"status": "Active",
"reason": None,
"tokenOwner": None,
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assertTrue(res["OK"])
self.assertEqual(res["Value"]["policyCombinedResult"]["Status"],
"Banned")
# site2
decisionParams = {
"element": "Site",
"name": "Site2",
"elementType": "CE",
"statusType": "ReadAccess",
"status": "Active",
"domain": "test",
"reason": None,
"tokenOwner": None,
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assertTrue(res["OK"])
self.assertEqual(res["Value"]["policyCombinedResult"]["Status"],
"Banned")
# mySE
decisionParams = {
"element": "Resource",
"name": "mySE",
"elementType": "StorageElement",
"statusType": "ReadAccess",
"status": "Active",
"reason": None,
"tokenOwner": None,
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assertTrue(res["OK"])
self.assertEqual(res["Value"]["policyCombinedResult"]["Status"],
"Active")
# SE1
decisionParams = {
"element": "Resource",
"name": "SE1",
"elementType": "StorageElement",
"statusType": "ReadAccess",
"status": "Active",
"reason": None,
"tokenOwner": None,
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assertTrue(res["OK"])
self.assertEqual(res["Value"]["policyCombinedResult"]["Status"],
"Banned")
def test_site(self):
pdp = PDP()
# empty
pdp.setup(None)
res = pdp.takeDecision()
self.assert_(res['OK'])
# site
decisionParams = {
'element': 'Site',
'name': 'Site1',
'elementType': None,
'statusType': 'ReadAccess',
'status': 'Active',
'reason': None,
'tokenOwner': None
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assert_(res['OK'])
self.assertEqual(res['Value']['policyCombinedResult']['Status'],
'Banned')
# mySE
decisionParams = {
'element': 'Resource',
'name': 'mySE',
'elementType': 'StorageElement',
'statusType': 'ReadAccess',
'status': 'Active',
'reason': None,
'tokenOwner': None
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assert_(res['OK'])
self.assertEqual(res['Value']['policyCombinedResult']['Status'],
'Active')
# SE1
decisionParams = {
'element': 'Resource',
'name': 'SE1',
'elementType': 'StorageElement',
'statusType': 'ReadAccess',
'status': 'Active',
'reason': None,
'tokenOwner': None
}
pdp.setup(decisionParams)
res = pdp.takeDecision()
self.assert_(res['OK'])
self.assertEqual(res['Value']['policyCombinedResult']['Status'],
'Banned')
def test__policyCombination(self):
for granularity in ValidRes:
for status in ValidStatus:
for oldStatus in ValidStatus:
if status == oldStatus:
continue
sm = StateMachine(self.VO, status)
for newStatus1 in ValidStatus:
for newStatus2 in ValidStatus:
pdp = PDP(self.VO, granularity, 'XX', status, oldStatus, 'XX')
polRes = {'Status':newStatus1, 'Reason':'-Reason1-'}
polRes2 = {'Status':newStatus2, 'Reason':'-Reason2-'}
# 0 policies
res = pdp._policyCombination([])
self.assertEqual(res, {})
# 1 policy
res = pdp._policyCombination([polRes])
if sm.transitionAllowed(newStatus1):
self.assertEqual(res['Status'], newStatus1)
self.assertEqual(res['Reason'], '-Reason1-')
else:
self.assertEqual(res, {})
# 2 policies
res = pdp._policyCombination([polRes, polRes2])
if sm.valueOfPolicy(polRes) > sm.valueOfPolicy(polRes2):
if sm.transitionAllowed(newStatus2):
self.assertEqual(res['Status'], newStatus2)
self.assertEqual(res['Reason'], '-Reason2-')
elif sm.transitionAllowed(newStatus1):
self.assertEqual(res['Status'], newStatus1)
self.assertEqual(res['Reason'], '-Reason1-')
else:
self.assertEqual(res, {})
elif sm.valueOfPolicy(polRes) < sm.valueOfPolicy(polRes2):
if sm.transitionAllowed(newStatus1):
self.assertEqual(res['Status'], newStatus1)
self.assertEqual(res['Reason'], '-Reason1-')
elif sm.transitionAllowed(newStatus2):
self.assertEqual(res['Status'], newStatus2)
self.assertEqual(res['Reason'], '-Reason2-')
else:
self.assertEqual(res, {})
else:
if sm.transitionAllowed(newStatus1):
self.assertEqual(res['Status'], newStatus1)
self.assertEqual(res['Reason'], '-Reason1- |###| -Reason2-')
else:
self.assertEqual(res, {})
# all different policies
def make_polres(status):
return { 'Status': status, 'Reason': 'Because of ' + status }
all_polres = map(make_polres, ValidStatus)
res = pdp._policyCombination(all_polres)
self.assertEqual(res['Status'], 'Banned')
def enforce(self,
pdpIn=None,
rsDBIn=None,
rmDBIn=None,
ncIn=None,
setupIn=None,
daIn=None,
csAPIIn=None,
knownInfo=None):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP(self.VOExtension,
granularity=self.__granularity,
name=self.__name,
status=self.__status,
formerStatus=self.__formerStatus,
reason=self.__reason,
siteType=self.__siteType,
serviceType=self.__serviceType,
resourceType=self.__resourceType,
useNewRes=self.useNewRes)
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = CS.getSetup()['Value']
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
###################
# policy decision #
###################
resDecisions = pdp.takeDecision(knownInfo=knownInfo)
assert (type(resDecisions) == dict and resDecisions != {})
res = resDecisions['PolicyCombinedResult']
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyPolTypeActions(self.__granularity, self.__name, resDecisions,
res)
else:
policyType = res['PolicyType']
if 'Resource_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Resource_PolType",
self.VOExtension)
m.ResourcePolTypeActions(self.__granularity, self.__name,
resDecisions, res, rsDB, rmDB)
if 'Alarm_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Alarm_PolType",
self.VOExtension)
m.AlarmPolTypeActions(self.__name,
res,
nc,
setup,
rsDB,
rmDB,
Granularity=self.__granularity,
SiteType=self.__siteType,
ServiceType=self.__serviceType,
ResourceType=self.__resourceType)
if 'RealBan_PolType' in policyType and self.__realBan == True:
m = Utils.voimport(actionBaseMod + ".RealBan_PolType",
self.VOExtension)
m.RealBanPolTypeActions(self.__granularity, self.__name, res,
da, csAPI, setup)
def execute(self):
"""
The main TokenAgent execution method.
Checks for tokens owned by users that are expiring, and notifies those users.
Calls rsDB.setToken() to set 'RS_SVC' as owner for those tokens that expired.
"""
adminMail = ''
try:
#reAssign the token to RS_SVC
for g in self.ELEMENTS:
tokensExpired = self.rsDB.getTokens(g, None,
datetime.datetime.utcnow())
if tokensExpired:
adminMail += '\nLIST OF EXPIRED TOKENS\n'
for token in tokensExpired:
name = token[0]
user = token[1]
self.rsDB.setToken(
g, name, 'RS_SVC',
datetime.datetime(9999, 12, 31, 23, 59, 59))
adminMail += ' %s %s\n' % (user.ljust(20), name)
#notify token owners
inNHours = datetime.datetime.utcnow() + datetime.timedelta(
hours=self.notifyHours)
for g in self.ELEMENTS:
tokensExpiring = self.rsDB.getTokens(g, None, inNHours)
if tokensExpiring:
adminMail += '\nLIST OF EXPIRING TOKENS\n'
for token in tokensExpiring:
name = token[0]
user = token[1]
adminMail += '\n %s %s\n' % (user.ljust(20), name)
if user == 'RS_SVC':
continue
pdp = PDP(self.VOExt, granularity=g, name=name)
decision = pdp.takeDecision()
pcresult = decision['PolicyCombinedResult']
spresult = decision['SinglePolicyResults']
expiration = token[2]
mailMessage = "The token for %s %s " % (g, name)
mailMessage = mailMessage + "will expire on %s\n\n" % expiration
mailMessage = mailMessage + "You can renew it with command 'dirac-rss-renew-token'.\n"
mailMessage = mailMessage + "If you don't take any action, RSS will take control of the resource.\n\n"
policyMessage = ''
if pcresult:
policyMessage += " Policies applied will set status to %s.\n" % pcresult[
'Status']
for spr in spresult:
policyMessage += " %s Status->%s\n" % (
spr['PolicyName'].ljust(25), spr['Status'])
mailMessage += policyMessage
adminMail += policyMessage
self.nc.sendMail(
getMailForUser(user)['Value'][0],
'Token for %s is expiring' % name, mailMessage)
if adminMail != '':
self.nc.sendMail(
getMailForUser('ubeda')['Value'][0], "Token's summary",
adminMail)
return S_OK()
except Exception:
errorStr = "TokenAgent execution"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
#############################################################################
class PEP:
'''
PEP (Policy Enforcement Point) initialization
:params:
:attr:`granularity` : string - a ValidElement (optional)
:attr:`name` : string - optional name (e.g. of a site)
:attr:`status` : string - optional status
:attr:`formerStatus` : string - optional former status
:attr:`reason` : string - optional reason for last status change
:attr:`siteType` : string - optional site type
:attr:`serviceType` : string - optional service type
:attr:`resourceType` : string - optional resource type
:attr:`futureEnforcement` : optional
[
{
'PolicyType': a PolicyType
'Granularity': a ValidElement (optional)
}
]
'''
def __init__( self, pdp = None, clients = None ):
'''
Enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdp` : a custom PDP object (optional)
:attr:`clients` : a dictionary containing modules corresponding to clients.
'''
if clients is None:
clients = {}
try:
self.rsClient = clients[ 'ResourceStatusClient' ]
except KeyError:
self.rsClient = ResourceStatusClient()
try:
self.rmClient = clients[ 'ResourceManagementClient' ]
except KeyError:
self.rmClient = ResourceManagementClient()
self.clients = clients
if not pdp:
self.pdp = PDP( **clients )
def enforce( self, granularity = None, name = None, statusType = None,
status = None, formerStatus = None, reason = None,
siteType = None, serviceType = None, resourceType = None,
tokenOwner = None, useNewRes = False, knownInfo = None ):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## real ban flag #########################################################
realBan = False
if tokenOwner is not None:
if tokenOwner == 'RS_SVC':
realBan = True
## sanitize input ##########################################################
## IS IT REALLY NEEDED ??
validElements = RssConfiguration.getValidElements()
if granularity is not None and granularity not in validElements:
return S_ERROR( 'Granularity "%s" not valid' % granularity )
validStatusTypes = RssConfiguration.getValidStatusTypes()
if statusType is not None and statusType not in validStatusTypes[ granularity ]['StatusType']:
return S_ERROR( 'StatusType "%s" not valid' % statusType )
validStatus = RssConfiguration.getValidStatus()
if status is not None and status not in validStatus:
return S_ERROR( 'Status "%s" not valid' % status )
validStatus = RssConfiguration.getValidStatus()
if formerStatus is not None and formerStatus not in validStatus:
return S_ERROR( 'FormerStatus "%s" not valid' % formerStatus )
validSiteTypes = RssConfiguration.getValidSiteTypes()
if siteType is not None and siteType not in validSiteTypes:
return S_ERROR( 'SiteType "%s" not valid' % siteType )
validServiceTypes = RssConfiguration.getValidServiceTypes()
if serviceType is not None and serviceType not in validServiceTypes:
return S_ERROR( 'ServiceType "%s" not valid' % serviceType )
validResourceTypes = RssConfiguration.getValidResourceTypes()
if resourceType is not None and resourceType not in validResourceTypes:
return S_ERROR( 'ResourceType "%s" not valid' % resourceType )
## policy setup ############################################################
self.pdp.setup( granularity = granularity, name = name,
statusType = statusType, status = status,
formerStatus = formerStatus, reason = reason,
siteType = siteType, serviceType = serviceType,
resourceType = resourceType, useNewRes = useNewRes )
## policy decision #########################################################
resDecisions = self.pdp.takeDecision( knownInfo = knownInfo )
## record all results before doing anything else
for resP in resDecisions[ 'SinglePolicyResults' ]:
if not resP.has_key( 'OLD' ):
self.clients[ "rmClient" ].insertPolicyResultLog( granularity, name,
resP[ 'PolicyName' ],
statusType,
resP[ 'Status' ],
resP[ 'Reason' ], now )
else:
gLogger.warn( 'OLD: %s' % resP )
res = resDecisions[ 'PolicyCombinedResult' ]
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyAction(granularity, name, statusType, resDecisions).run()
else:
policyType = res[ 'PolicyType' ]
if 'Resource_PolType' in policyType:
action = Utils.voimport( '%s.ResourceAction' % actionBaseMod )
action.ResourceAction(granularity, name, statusType, resDecisions,
rsClient=self.rsClient,
rmClient=self.rmClient).run()
if 'Alarm_PolType' in policyType:
action = Utils.voimport( '%s.AlarmAction' % actionBaseMod )
action.AlarmAction(granularity, name, statusType, resDecisions,
Clients=self.clients,
Params={"Granularity" : granularity,
"SiteType" : siteType,
"ServiceType" : serviceType,
"ResourceType" : resourceType}).run()
if 'RealBan_PolType' in policyType and realBan:
action = Utils.voimport( '%s.RealBanAction' % actionBaseMod )
action.RealBanAction(granularity, name, resDecisions).run()
return resDecisions
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class PEP:
""" PEP ( Policy Enforcement Point )
"""
def __init__( self, clients = None ):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
if clients is None:
clients = {}
# PEP uses internally two of the clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' in clients:
self.rsClient = clients[ 'ResourceStatusClient' ]
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP( clients )
def enforce( self, decisionParams ):
""" Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
# Setup PDP with new parameters dictionary
self.pdp.setup( decisionParams )
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions[ 'OK' ]:
gLogger.error( 'PEP: Something went wrong, not enforcing policies for %s' % decisionParams )
return resDecisions
resDecisions = resDecisions[ 'Value' ]
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions[ 'decissionParams' ]
policyCombinedResult = resDecisions[ 'policyCombinedResult' ]
singlePolicyResults = resDecisions[ 'singlePolicyResults' ]
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated( decisionParams )
if not isNotUpdated[ 'OK' ]:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult[ 'PolicyAction' ]:
try:
actionMod = Utils.voimport( 'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' % policyActionType )
except ImportError:
gLogger.error( 'Error importing %s action' % policyActionType )
continue
try:
action = getattr( actionMod, policyActionType )
except AttributeError:
gLogger.error( 'Error importing %s action class' % policyActionType )
continue
actionObj = action( policyActionName, decisionParams, policyCombinedResult,
singlePolicyResults, self.clients )
gLogger.debug( ( policyActionName, policyActionType ) )
actionResult = actionObj.run()
if not actionResult[ 'OK' ]:
gLogger.error( actionResult[ 'Message' ] )
return S_OK( resDecisions )
def __isNotUpdated( self, decisionParams ):
""" Checks for the existence of the element as it was passed to the PEP. It may
happen that while being the element processed by the PEP an user through the
web interface or the CLI has updated the status for this particular element. As
a result, the PEP would overwrite whatever the user had set. This check is not
perfect, as still an user action can happen while executing the actions, but
the probability is close to 0. However, if there is an action that takes seconds
to be executed, this must be re-evaluated. !
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies
:return: S_OK / S_ERROR
"""
# Copy original dictionary and get rid of one key we cannot pass as kwarg
selectParams = decisionParams.copy()
del selectParams[ 'element' ]
del selectParams[ 'active' ]
# We expect to have an exact match. If not, then something has changed and
# we cannot proceed with the actions.
unchangedRow = self.rsClient.selectStatusElement( decisionParams[ 'element' ],
'Status', **selectParams )
if not unchangedRow[ 'OK' ]:
return unchangedRow
if not unchangedRow[ 'Value' ]:
msg = '%(name)s ( %(status)s / %(statusType)s ) has been updated after PEP started running'
return S_ERROR( msg % selectParams )
return S_OK()
#...............................................................................
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def execute(self):
'''
The main TokenAgent execution method.
Checks for tokens owned by users that are expiring, and notifies those users.
Calls rsClient.setToken() to set 'RS_SVC' as owner for those tokens that expired.
'''
adminMail = ''
try:
reason = 'Out of date token'
#reAssign the token to RS_SVC
#for g in self.ELEMENTS:
validElements = RssConfiguration.getValidElements()
for granularity in validElements:
tokensExpired = self.rsClient.getTokens(
granularity, tokenExpiration=datetime.datetime.utcnow())
if tokensExpired['Value']:
adminMail += '\nLIST OF EXPIRED %s TOKENS\n' % granularity
adminMail += '%s|%s|%s\n' % (
'user'.ljust(20), 'name'.ljust(15), 'status type')
for token in tokensExpired['Value']:
name = token[1]
stype = token[2]
user = token[9]
self.rsClient.setToken(
granularity, name, stype, reason, 'RS_SVC',
datetime.datetime(9999, 12, 31, 23, 59, 59))
adminMail += ' %s %s %s\n' % (user.ljust(20),
name.ljust(15), stype)
#notify token owners
inNHours = datetime.datetime.utcnow() + datetime.timedelta(
hours=self.notifyHours)
#for g in self.ELEMENTS:
for granularity in validElements:
tokensExpiring = self.rsClient.getTokens(
granularity, tokenExpiration=inNHours)
if tokensExpiring['Value']:
adminMail += '\nLIST OF EXPIRING %s TOKENS\n' % granularity
adminMail += '%s|%s|%s\n' % (
'user'.ljust(20), 'name'.ljust(15), 'status type')
for token in tokensExpiring['Value']:
name = token[1]
stype = token[2]
user = token[9]
adminMail += '\n %s %s %s\n' % (user.ljust(20),
name.ljust(15), stype)
#If user is RS_SVC, we ignore this, whenever the token is out, this
#agent will set again the token to RS_SVC
if user == 'RS_SVC':
continue
pdp = PDP(granularity=granularity,
name=name,
statusType=stype)
decision = pdp.takeDecision()
pcresult = decision['PolicyCombinedResult']
spresult = decision['SinglePolicyResults']
expiration = token[10]
mailMessage = "The token for %s %s ( %s )" % (granularity,
name, stype)
mailMessage = mailMessage + " will expire on %s\n\n" % expiration
mailMessage = mailMessage + "You can renew it with command 'dirac-rss-renew-token'.\n"
mailMessage = mailMessage + "If you don't take any action, RSS will take control of the resource.\n\n"
policyMessage = ''
if pcresult['Action']:
policyMessage += " Policies applied will set status to %s.\n" % pcresult[
'Status']
for spr in spresult:
policyMessage += " %s Status->%s\n" % (
spr['PolicyName'].ljust(25), spr['Status'])
mailMessage += policyMessage
adminMail += policyMessage
self.noClient.sendMail(
self.rmClient.getUserRegistryCache(user)['Value'][0]
[2], 'Token for %s is expiring' % name, mailMessage)
if adminMail != '':
#FIXME: 'ubeda' is not generic ;p
self.noClient.sendMail(
self.rmClient.getUserRegistryCache('ubeda')['Value'][0][2],
"Token's summary", adminMail)
return S_OK()
except Exception:
errorStr = "TokenAgent execution"
self.log.exception(errorStr)
return S_ERROR(errorStr)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
#risorsa = {'name':'ce106.pic.es', 'siteType':'T1', 'resourceType':'CE'} #WRONG
#risorsa2 = {'name':'srm-lhcb.cern.ch#', 'siteType':'T0', 'resourceType':'SE'} #WRONG
#se = {'name':'CERN_MC_M-DST#', 'siteType':'T0'} #WRONG
print "\n\n ~~~~~~~ SITO ~~~~~~~ %s \n" %(sito)
for status in ValidStatus:
# for oldStatus in ValidStatus:
# if status == oldStatus:
# continue
print "############################"
print " "
print 'dans le test:', status#, oldStatus
pdp = PDP(VO, granularity = 'Site', name = sito['name'], status = status,
# formerStatus = oldStatus,
reason = 'XXXXX', siteType = sito['siteType'],
useNewRes = useNewRes
)
res = pdp.takeDecision()
print res
#print "\n\n ~~~~~~~ SERVICE 1 ~~~~~~~ : %s \n " %servizio
#
#for status in ValidStatus:
## for oldStatus in ValidStatus:
## if status == oldStatus:
## continue
# print "############################"
# print " "
# print 'nel test:', status#, oldStatus
# pdp = PDP(VO, granularity = 'Service', name = servizio['name'], status = status,
#risorsa = {'name':'ce106.pic.es', 'siteType':'T1', 'resourceType':'CE'} #WRONG
#risorsa2 = {'name':'srm-lhcb.cern.ch#', 'siteType':'T0', 'resourceType':'SE'} #WRONG
#se = {'name':'CERN_MC_M-DST#', 'siteType':'T0'} #WRONG
print "\n\n ~~~~~~~ SITO ~~~~~~~ %s \n" %(sito)
for status in ValidStatus:
# for oldStatus in ValidStatus:
# if status == oldStatus:
# continue
print "############################"
print " "
print 'dans le test:', status#, oldStatus
pdp = PDP(VO, granularity = 'Site', name = sito['name'], status = status,
# formerStatus = oldStatus,
reason = 'XXXXX', siteType = sito['siteType'],
useNewRes = useNewRes
)
res = pdp.takeDecision()
print res
#print "\n\n ~~~~~~~ SERVICE 1 ~~~~~~~ : %s \n " %servizio
#
#for status in ValidStatus:
## for oldStatus in ValidStatus:
## if status == oldStatus:
## continue
# print "############################"
# print " "
# print 'nel test:', status#, oldStatus
# pdp = PDP(VO, granularity = 'Service', name = servizio['name'], status = status,
class PEP:
'''
PEP (Policy Enforcement Point) initialization
:params:
:attr:`granularity` : string - a ValidElement (optional)
:attr:`name` : string - optional name (e.g. of a site)
:attr:`status` : string - optional status
:attr:`formerStatus` : string - optional former status
:attr:`reason` : string - optional reason for last status change
:attr:`siteType` : string - optional site type
:attr:`serviceType` : string - optional service type
:attr:`resourceType` : string - optional resource type
:attr:`futureEnforcement` : optional
[
{
'PolicyType': a PolicyType
'Granularity': a ValidElement (optional)
}
]
'''
def __init__(self, pdp=None, clients=None):
'''
Enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdp` : a custom PDP object (optional)
:attr:`clients` : a dictionary containing modules corresponding to clients.
'''
if clients is None:
clients = {}
try:
self.rsClient = clients['ResourceStatusClient']
except KeyError:
self.rsClient = ResourceStatusClient()
try:
self.rmClient = clients['ResourceManagementClient']
except KeyError:
self.rmClient = ResourceManagementClient()
self.clients = clients
if not pdp:
self.pdp = PDP(**clients)
def enforce(self,
granularity=None,
name=None,
statusType=None,
status=None,
formerStatus=None,
reason=None,
siteType=None,
serviceType=None,
resourceType=None,
tokenOwner=None,
useNewRes=False,
knownInfo=None):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## real ban flag #########################################################
realBan = False
if tokenOwner is not None:
if tokenOwner == 'RS_SVC':
realBan = True
## sanitize input ##########################################################
## IS IT REALLY NEEDED ??
validElements = RssConfiguration.getValidElements()
if granularity is not None and granularity not in validElements:
return S_ERROR('Granularity "%s" not valid' % granularity)
validStatusTypes = RssConfiguration.getValidStatusTypes()
if statusType is not None and statusType not in validStatusTypes[
granularity]['StatusType']:
return S_ERROR('StatusType "%s" not valid' % statusType)
validStatus = RssConfiguration.getValidStatus()
if status is not None and status not in validStatus:
return S_ERROR('Status "%s" not valid' % status)
validStatus = RssConfiguration.getValidStatus()
if formerStatus is not None and formerStatus not in validStatus:
return S_ERROR('FormerStatus "%s" not valid' % formerStatus)
validSiteTypes = RssConfiguration.getValidSiteTypes()
if siteType is not None and siteType not in validSiteTypes:
return S_ERROR('SiteType "%s" not valid' % siteType)
validServiceTypes = RssConfiguration.getValidServiceTypes()
if serviceType is not None and serviceType not in validServiceTypes:
return S_ERROR('ServiceType "%s" not valid' % serviceType)
validResourceTypes = RssConfiguration.getValidResourceTypes()
if resourceType is not None and resourceType not in validResourceTypes:
return S_ERROR('ResourceType "%s" not valid' % resourceType)
## policy setup ############################################################
self.pdp.setup(granularity=granularity,
name=name,
statusType=statusType,
status=status,
formerStatus=formerStatus,
reason=reason,
siteType=siteType,
serviceType=serviceType,
resourceType=resourceType,
useNewRes=useNewRes)
## policy decision #########################################################
resDecisions = self.pdp.takeDecision(knownInfo=knownInfo)
## record all results before doing anything else
for resP in resDecisions['SinglePolicyResults']:
if not resP.has_key('OLD'):
self.clients["rmClient"].insertPolicyResultLog(
granularity, name, resP['PolicyName'], statusType,
resP['Status'], resP['Reason'], now)
else:
gLogger.warn('OLD: %s' % resP)
res = resDecisions['PolicyCombinedResult']
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyAction(granularity, name, statusType, resDecisions).run()
else:
policyType = res['PolicyType']
if 'Resource_PolType' in policyType:
action = Utils.voimport('%s.ResourceAction' % actionBaseMod)
action.ResourceAction(granularity,
name,
statusType,
resDecisions,
rsClient=self.rsClient,
rmClient=self.rmClient).run()
if 'Alarm_PolType' in policyType:
action = Utils.voimport('%s.AlarmAction' % actionBaseMod)
action.AlarmAction(granularity,
name,
statusType,
resDecisions,
Clients=self.clients,
Params={
"Granularity": granularity,
"SiteType": siteType,
"ServiceType": serviceType,
"ResourceType": resourceType
}).run()
if 'RealBan_PolType' in policyType and realBan:
action = Utils.voimport('%s.RealBanAction' % actionBaseMod)
action.RealBanAction(granularity, name, resDecisions).run()
return resDecisions
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
