def __init__(self, parentLogger=None):
if not parentLogger:
parentLogger = gLogger
self.log = parentLogger.getSubLogger(self.__class__.__name__)
result = getDBParameters("WorkloadManagement/PilotsLoggingDB")
if not result["OK"]:
raise RuntimeError("Cannot get database parameters: %s" % result["Message"])
dbParameters = result["Value"]
self.dbHost = dbParameters["Host"]
self.dbPort = dbParameters["Port"]
self.dbUser = dbParameters["User"]
self.dbPass = dbParameters["Password"]
self.dbName = dbParameters["DBName"]
# These are the list of tables that will be created.
# They can be extended in an extension module
self.tablesList = getattr(Utils.voimport("DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB"), "TABLESLIST")
self.__initializeConnection()
resp = self.__initializeDB()
if not resp["OK"]:
raise Exception("Couldn't create tables: " + resp["Message"])
def initializeResourceStatusHandler(_serviceInfo):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
# Publisher is on boxes right now
#
# rmDB = ResourceStatusDB()
# cc = CommandCaller()
# global VOExtension
# VOExtension = getExt()
# ig = InfoGetter( VOExtension )
# WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
# global publisher
# publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
# infoGetterIn = ig, WMSAdminIn = WMSAdmin )
syncModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Utilities.Synchronizer')
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent(syncObject.sync)
return S_OK()
def initializeResourceStatusHandler( _serviceInfo ):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
# Publisher is on boxes right now
#
# rmDB = ResourceStatusDB()
# cc = CommandCaller()
# global VOExtension
# VOExtension = getExt()
# ig = InfoGetter( VOExtension )
# WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
# global publisher
# publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
# infoGetterIn = ig, WMSAdminIn = WMSAdmin )
syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer' )
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent( syncObject.sync )
return S_OK()
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def __init__(self):
self.log = gLogger.getSubLogger('PilotsLoggingDB')
result = getDBParameters('WorkloadManagement/PilotsLoggingDB')
if not result['OK']:
raise RuntimeError('Cannot get database parameters: %s' %
result['Message'])
dbParameters = result['Value']
self.dbHost = dbParameters['Host']
self.dbPort = dbParameters['Port']
self.dbUser = dbParameters['User']
self.dbPass = dbParameters['Password']
self.dbName = dbParameters['DBName']
# These are the list of tables that will be created.
# They can be extended in an extension module
self.tablesList = getattr(
Utils.voimport(
'DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB'),
'TABLESLIST')
self.__initializeConnection()
resp = self.__initializeDB()
if not resp['OK']:
raise Exception("Couldn't create tables: " + resp['Message'])
def enforce(self, decissionParams):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## policy decision point setup #############################################
self.pdp.setup(decissionParams)
## policy decision #########################################################
resDecisions = self.pdp.takeDecision()
if not resDecisions['OK']:
gLogger.error(
'PEP: Something went wrong, not enforcing policies for %s' %
decissionParams)
return resDecisions
resDecisions = resDecisions['Value']
# We take from PDP the decision parameters used to find the policies
decissionParams = resDecisions['decissionParams']
policyCombinedResult = resDecisions['policyCombinedResult']
singlePolicyResults = resDecisions['singlePolicyResults']
for policyActionName, policyActionType in policyCombinedResult[
'PolicyAction']:
try:
actionMod = Utils.voimport(
'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' %
policyActionType)
except ImportError:
gLogger.error('Error importing %s action' % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
gLogger.error('Error importing %s action class' %
policyActionType)
continue
actionObj = action(policyActionName, decissionParams,
policyCombinedResult, singlePolicyResults,
self.clients)
gLogger.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult['OK']:
gLogger.error(actionResult['Message'])
return S_OK(resDecisions)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( 'ResourceStatusDB' )
#These are the list of tables that will be created.
#They can be extended in an extension module
self.tablesList = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceStatusDB' ),
'TABLESLIST')
self.tablesListWithID = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceStatusDB' ),
'TABLESLISTWITHID')
self.extensions = gConfig.getValue( 'DIRAC/Extensions', [] )
self.__initializeConnection( 'ResourceStatus/ResourceStatusDB' )
self.__initializeDB()
def policyInvocation( self, granularity = None, name = None,
status = None, policy = None, args = None, pName = None,
pModule = None, extraArgs = None, commandIn = None ):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if not policy:
try:
policyModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.%s' % pModule )
except ImportError:
_msg = 'Unable to import a policy module named %s, falling back on AlwaysFalse_Policy.' % pModule
gLogger.warn( _msg )
policyModule = __import__( 'DIRAC.ResourceStatusSystem.Policy.AlwaysFalse_Policy',
globals(), locals(), ['*'] )
pModule = 'AlwaysFalse_Policy'
try:
policy = getattr( policyModule, pModule )()
except AttributeError as exc:
print policyModule, pModule
raise exc
if not args:
args = ( granularity, name )
if extraArgs:
args = args + tuple( extraArgs )
if commandIn:
commandIn = self.cCaller.setCommandObject( commandIn )
for clientName, clientInstance in self.clients.items():
self.cCaller.setAPI( commandIn, clientName, clientInstance )
res = self._innerEval( policy, args, commandIn = commandIn )
# Just adding the PolicyName to the result of the evaluation of the policy
res[ 'PolicyName' ] = pName
return res
def __init__(self, VOExtension):
"""
Standard constructor
:params:
:attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
"""
configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
self.C_Policies = copy.deepcopy(configModule.Policies)
def policyInvocation(self, decisionParams, policyDict):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if 'module' not in policyDict:
return S_ERROR('Malformed policyDict %s' % policyDict)
pModuleName = policyDict['module']
if 'command' not in policyDict:
return S_ERROR('Malformed policyDict %s' % policyDict)
pCommand = policyDict['command']
if 'args' not in policyDict:
return S_ERROR('Malformed policyDict %s' % policyDict)
pArgs = policyDict['args']
try:
policyModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName)
except ImportError:
return S_ERROR(
'Unable to import DIRAC.ResourceStatusSystem.Policy.%s' %
pModuleName)
if not hasattr(policyModule, pModuleName):
return S_ERROR('%s has no attibute %s' %
(policyModule, pModuleName))
policy = getattr(policyModule, pModuleName)()
command = self.cCaller.commandInvocation(pCommand, pArgs,
decisionParams, self.clients)
if not command['OK']:
return command
command = command['Value']
evaluationResult = self.policyEvaluation(policy, command)
if evaluationResult['OK']:
evaluationResult['Value']['Policy'] = policyDict
return evaluationResult
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
WMSAdminIn = None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool( 2, 5 )
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def __init__( self ):
""" Constructor. Imports the policy configurations containing the command
information, among other things.
examples:
>>> iGetter = InfoGetter()
"""
configModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.Configurations' )
self.policies = copy.deepcopy( configModule.POLICIESMETA )
def __init__(self, VOExtension):
"""
Standard constructor
:params:
:attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
"""
configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
self.C_Policies = copy.deepcopy(configModule.Policies)
def policyInvocation( self, decisionParams, policyDict ):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if not 'module' in policyDict:
return S_ERROR( 'Malformed policyDict %s' % policyDict )
pModuleName = policyDict[ 'module' ]
if not 'command' in policyDict:
return S_ERROR( 'Malformed policyDict %s' % policyDict )
pCommand = policyDict[ 'command' ]
if not 'args' in policyDict:
return S_ERROR( 'Malformed policyDict %s' % policyDict )
pArgs = policyDict[ 'args' ]
try:
policyModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName )
except ImportError:
return S_ERROR( 'Unable to import DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName )
if not hasattr( policyModule, pModuleName ):
return S_ERROR( '%s has no attibute %s' % ( policyModule, pModuleName ) )
policy = getattr( policyModule, pModuleName )()
command = self.cCaller.commandInvocation( pCommand, pArgs, decisionParams, self.clients )
if not command[ 'OK' ]:
return command
command = command[ 'Value' ]
evaluationResult = self.policyEvaluation( policy, command )
if evaluationResult[ 'OK' ]:
evaluationResult[ 'Value' ][ 'Policy' ] = policyDict
return evaluationResult
def initializeResourceStatusHandler( _serviceInfo ):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer' )
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent( syncObject.sync )
return S_OK()
def __init__(self):
"""c'tor
:param self: self reference
"""
super(ResourceStatusDB, self).__init__()
# These are the list of tables that will be created.
# They can be extended in an extension module
self.tablesList = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"),
"TABLESLIST")
self.tablesListWithID = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"),
"TABLESLISTWITHID")
self.extensions = gConfig.getValue("DIRAC/Extensions", [])
self._initializeConnection("ResourceStatus/ResourceStatusDB")
# Create required tables
self._createTablesIfNotThere(self.tablesList)
self._createTablesIfNotThere(self.tablesListWithID)
def __init__(self):
"""c'tor
:param self: self reference
"""
super(ResourceManagementDB, self).__init__()
# This is the list of tables that will be created.
# It can be extended in an extension module
self.tablesList = getattr(Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceManagementDB"), "TABLESLIST")
self._initializeConnection("ResourceStatus/ResourceManagementDB")
# Create required tables
self._createTablesIfNotThere(self.tablesList)
def __loadTestObj(self):
_module_pre = 'DIRAC.ResourceStatusSystem.SAM.SAMTest.'
for testType, testDict in self.tests.items():
moduleName = testDict[ 'module' ]
args = testDict.get( 'args', {} )
args.update( testDict[ 'match' ] )
args[ 'TestType' ] = testType
try:
testModule = Utils.voimport( _module_pre + moduleName )
except ImportError, e:
gLogger.error( "Unable to import %s, %s" % ( _module_pre + moduleName, e ) )
continue
testClass = getattr( testModule, moduleName )
obj = testClass(args, self.apis)
testDict[ 'object' ] = obj
def commandInvocation(commandTuple,
pArgs=None,
decissionParams=None,
clients=None):
'''
Returns a command object, given commandTuple
:params:
`commandTuple`: a tuple, where commandTuple[0] is a module name and
commandTuple[1] is a class name (inside the module)
'''
if commandTuple is None:
return S_OK(None)
# decission params can be a dictionary passed with all the element parameters
# used mostly by the PDP to inject all relevant information
if decissionParams is None:
decissionParams = {}
# arguments hardcoded on Configurations.py for the policy
if pArgs is None:
pArgs = {}
try:
cModule = commandTuple[0]
cClass = commandTuple[1]
commandModule = Utils.voimport('DIRAC.ResourceStatusSystem.Command.' +
cModule)
except ImportError:
return S_ERROR("Import error for command %s." % (cModule))
if not hasattr(commandModule, cClass):
return S_ERROR('%s has no %s' % (cModule, cClass))
# We merge decision parameters and policy arguments.
newArgs = copy.deepcopy(decissionParams)
newArgs.update(pArgs)
commandObject = getattr(commandModule, cClass)(newArgs, clients)
return S_OK(commandObject)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __loadTestObj(self):
_module_pre = 'DIRAC.ResourceStatusSystem.SAM.SAMTest.'
for testType, testDict in self.tests.items():
moduleName = testDict['module']
args = testDict.get('args', {})
args.update(testDict['match'])
args['TestType'] = testType
try:
testModule = Utils.voimport(_module_pre + moduleName)
except ImportError, e:
gLogger.error("Unable to import %s, %s" %
(_module_pre + moduleName, e))
continue
testClass = getattr(testModule, moduleName)
obj = testClass(args, self.apis)
testDict['object'] = obj
def setCommandObject( self, comm ):
"""
Returns a command object, given comm
:params:
`comm`: a tuple, where comm[0] is a module name and comm[1] is a class name (inside the module)
"""
try:
cModule = comm[0]
cClass = comm[1]
commandModule = Utils.voimport("DIRAC.ResourceStatusSystem.Command." + cModule)
except ImportError:
gLogger.warn("Command %s/%s not found, using dummy command DoNothing_Command." % (cModule, cClass))
cClass = "DoNothing_Command"
commandModule = __import__("DIRAC.ResourceStatusSystem.Command.DoNothing_Command", globals(), locals(), ['*'])
c = getattr(commandModule, cClass)()
return c
def commandInvocation( commandTuple, pArgs = None, decissionParams = None, clients = None ):
'''
Returns a command object, given commandTuple
:params:
`commandTuple`: a tuple, where commandTuple[0] is a module name and
commandTuple[1] is a class name (inside the module)
'''
if commandTuple is None:
return S_OK( None )
# decission params can be a dictionary passed with all the element parameters
# used mostly by the PDP to inject all relevant information
if decissionParams is None:
decissionParams = {}
# arguments hardcoded on Configurations.py for the policy
if pArgs is None:
pArgs = {}
try:
cModule = commandTuple[ 0 ]
cClass = commandTuple[ 1 ]
commandModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Command.' + cModule )
except ImportError:
return S_ERROR( "Import error for command %s." % ( cModule ) )
if not hasattr( commandModule, cClass ):
return S_ERROR( '%s has no %s' % ( cModule, cClass ) )
# We merge decision parameters and policy arguments.
newArgs = copy.deepcopy( decissionParams )
newArgs.update( pArgs )
commandObject = getattr( commandModule, cClass )( newArgs, clients )
return S_OK( commandObject )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def main():
global subLogger
global ResourceManagementClient
subLogger = gLogger.getSubLogger(__file__)
# Script initialization
registerSwitches()
registerUsageMessage()
args, switchDict = parseSwitches()
ResourceManagementClient = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient"),
"ResourceManagementClient",
)
# Run script
run(args, switchDict)
# Bye
DIRACExit(0)
def __init__( self ):
self.log = gLogger.getSubLogger( 'PilotsLoggingDB' )
result = getDBParameters( 'WorkloadManagement/PilotsLoggingDB' )
if not result['OK']:
raise RuntimeError( 'Cannot get database parameters: %s' % result['Message'] )
dbParameters = result['Value']
self.dbHost = dbParameters['Host']
self.dbPort = dbParameters['Port']
self.dbUser = dbParameters['User']
self.dbPass = dbParameters['Password']
self.dbName = dbParameters['DBName']
#These are the list of tables that will be created.
#They can be extended in an extension module
self.tablesList = getattr(Utils.voimport( 'DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB' ),
'TABLESLIST')
self.__initializeConnection()
resp = self.__initializeDB( )
if not resp['OK']:
raise Exception( "Couldn't create tables: " + resp['Message'] )
def setCommandObject(self, comm):
"""
Returns a command object, given comm
:params:
`comm`: a tuple, where comm[0] is a module name and comm[1] is a class name (inside the module)
"""
try:
cModule = comm[0]
cClass = comm[1]
commandModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Command." + cModule)
except ImportError:
gLogger.warn(
"Command %s/%s not found, using dummy command DoNothing_Command."
% (cModule, cClass))
cClass = "DoNothing_Command"
commandModule = __import__(
"DIRAC.ResourceStatusSystem.Command.DoNothing_Command",
globals(), locals(), ['*'])
c = getattr(commandModule, cClass)()
return c
def policyInvocation(self,
granularity=None,
name=None,
status=None,
policy=None,
args=None,
pName=None,
pModule=None,
extraArgs=None,
commandIn=None):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if not policy:
try:
policyModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Policy.%s' % pModule)
except ImportError:
_msg = 'Unable to import a policy module named %s, falling back on AlwaysFalse_Policy.' % pModule
gLogger.warn(_msg)
policyModule = __import__(
'DIRAC.ResourceStatusSystem.Policy.AlwaysFalse_Policy',
globals(), locals(), ['*'])
pModule = 'AlwaysFalse_Policy'
try:
policy = getattr(policyModule, pModule)()
except AttributeError as exc:
print policyModule, pModule
raise exc
if not args:
args = (granularity, name)
if extraArgs:
args = args + tuple(extraArgs)
if commandIn:
commandIn = self.cCaller.setCommandObject(commandIn)
for clientName, clientInstance in self.clients.items():
self.cCaller.setAPI(commandIn, clientName, clientInstance)
res = self._innerEval(policy, args, commandIn=commandIn)
# Just adding the PolicyName to the result of the evaluation of the policy
res['PolicyName'] = pName
return res
''' ResourceManagementHandler
Module that allows users to access the ResourceManagementDB remotely.
'''
from DIRAC import gConfig, S_OK, gLogger
from DIRAC.Core.DISET.RequestHandler import RequestHandler
from DIRAC.ResourceStatusSystem.Utilities import Synchronizer, Utils
ResourceManagementDB = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceManagementDB' ),'ResourceManagementDB')
__RCSID__ = '$Id: $'
def initializeResourceManagementHandler( _serviceInfo ):
'''
Handler initialization, where we set the ResourceManagementDB as global db.
'''
global db
db = ResourceManagementDB()
# Regenerates DB tables if needed
db._checkTable()
syncObject = Synchronizer.Synchronizer()
gConfig.addListenerToNewVersionEvent( syncObject.sync )
return S_OK()
################################################################################
class ResourceManagementHandler( RequestHandler ):
def __init__(self):
configModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Policy.Configurations')
self.policies = copy.deepcopy(configModule.POLICIESMETA)
# $HeadURL $
""" ResourceManagementHandler
Module that allows users to access the ResourceManagementDB remotely.
"""
from DIRAC import gConfig, S_OK, gLogger
from DIRAC.Core.DISET.RequestHandler import RequestHandler
from DIRAC.ResourceStatusSystem.Utilities import Synchronizer, Utils
ResourceManagementDB = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceManagementDB"), "ResourceManagementDB"
)
__RCSID__ = "$Id: $"
db = False
def initializeResourceManagementHandler(_serviceInfo):
"""
Handler initialization, where we set the ResourceManagementDB as global db.
"""
global db
db = ResourceManagementDB()
# Regenerates DB tables if needed
db._checkTable()
syncObject = Synchronizer.Synchronizer()
gConfig.addListenerToNewVersionEvent(syncObject.sync)
Module that updates the RSS database ( ResourceStatusDB ) with the information
in the Resources section. If there are additions in the CS, those are incorporated
to the DB. If there are deletions, entries in RSS tables for those elements are
deleted ( except the Logs table ).
'''
__RCSID__ = '$Id: $'
from DIRAC import gLogger, S_OK
from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient
from DIRAC.ResourceStatusSystem.Utilities import CSHelpers
from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration
from DIRAC.ResourceStatusSystem.Utilities import Utils
ResourceManagementClient = getattr(
Utils.voimport(
'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient'),
'ResourceManagementClient')
class Synchronizer(object):
'''
Every time there is a successful write on the CS, Synchronizer().sync() is
executed. It updates the database with the values on the CS.
'''
def __init__(self, rStatus=None, rManagement=None):
# Warm up local CS
CSHelpers.warmUp()
if rStatus is None:
def enforce(self,
granularity=None,
name=None,
statusType=None,
status=None,
formerStatus=None,
reason=None,
siteType=None,
serviceType=None,
resourceType=None,
tokenOwner=None,
useNewRes=False,
knownInfo=None):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## real ban flag #########################################################
realBan = False
if tokenOwner is not None:
if tokenOwner == 'RS_SVC':
realBan = True
## sanitize input ##########################################################
## IS IT REALLY NEEDED ??
validElements = RssConfiguration.getValidElements()
if granularity is not None and granularity not in validElements:
return S_ERROR('Granularity "%s" not valid' % granularity)
validStatusTypes = RssConfiguration.getValidStatusTypes()
if statusType is not None and statusType not in validStatusTypes[
granularity]['StatusType']:
return S_ERROR('StatusType "%s" not valid' % statusType)
validStatus = RssConfiguration.getValidStatus()
if status is not None and status not in validStatus:
return S_ERROR('Status "%s" not valid' % status)
validStatus = RssConfiguration.getValidStatus()
if formerStatus is not None and formerStatus not in validStatus:
return S_ERROR('FormerStatus "%s" not valid' % formerStatus)
validSiteTypes = RssConfiguration.getValidSiteTypes()
if siteType is not None and siteType not in validSiteTypes:
return S_ERROR('SiteType "%s" not valid' % siteType)
validServiceTypes = RssConfiguration.getValidServiceTypes()
if serviceType is not None and serviceType not in validServiceTypes:
return S_ERROR('ServiceType "%s" not valid' % serviceType)
validResourceTypes = RssConfiguration.getValidResourceTypes()
if resourceType is not None and resourceType not in validResourceTypes:
return S_ERROR('ResourceType "%s" not valid' % resourceType)
## policy setup ############################################################
self.pdp.setup(granularity=granularity,
name=name,
statusType=statusType,
status=status,
formerStatus=formerStatus,
reason=reason,
siteType=siteType,
serviceType=serviceType,
resourceType=resourceType,
useNewRes=useNewRes)
## policy decision #########################################################
resDecisions = self.pdp.takeDecision(knownInfo=knownInfo)
## record all results before doing anything else
for resP in resDecisions['SinglePolicyResults']:
if not resP.has_key('OLD'):
self.clients["rmClient"].insertPolicyResultLog(
granularity, name, resP['PolicyName'], statusType,
resP['Status'], resP['Reason'], now)
else:
gLogger.warn('OLD: %s' % resP)
res = resDecisions['PolicyCombinedResult']
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyAction(granularity, name, statusType, resDecisions).run()
else:
policyType = res['PolicyType']
if 'Resource_PolType' in policyType:
action = Utils.voimport('%s.ResourceAction' % actionBaseMod)
action.ResourceAction(granularity,
name,
statusType,
resDecisions,
rsClient=self.rsClient,
rmClient=self.rmClient).run()
if 'Alarm_PolType' in policyType:
action = Utils.voimport('%s.AlarmAction' % actionBaseMod)
action.AlarmAction(granularity,
name,
statusType,
resDecisions,
Clients=self.clients,
Params={
"Granularity": granularity,
"SiteType": siteType,
"ServiceType": serviceType,
"ResourceType": resourceType
}).run()
if 'RealBan_PolType' in policyType and realBan:
action = Utils.voimport('%s.RealBanAction' % actionBaseMod)
action.RealBanAction(granularity, name, resDecisions).run()
return resDecisions
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def enforce(self,
pdpIn=None,
rsDBIn=None,
rmDBIn=None,
ncIn=None,
setupIn=None,
daIn=None,
csAPIIn=None,
knownInfo=None):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP(self.VOExtension,
granularity=self.__granularity,
name=self.__name,
status=self.__status,
formerStatus=self.__formerStatus,
reason=self.__reason,
siteType=self.__siteType,
serviceType=self.__serviceType,
resourceType=self.__resourceType,
useNewRes=self.useNewRes)
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = CS.getSetup()['Value']
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
###################
# policy decision #
###################
resDecisions = pdp.takeDecision(knownInfo=knownInfo)
assert (type(resDecisions) == dict and resDecisions != {})
res = resDecisions['PolicyCombinedResult']
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyPolTypeActions(self.__granularity, self.__name, resDecisions,
res)
else:
policyType = res['PolicyType']
if 'Resource_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Resource_PolType",
self.VOExtension)
m.ResourcePolTypeActions(self.__granularity, self.__name,
resDecisions, res, rsDB, rmDB)
if 'Alarm_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Alarm_PolType",
self.VOExtension)
m.AlarmPolTypeActions(self.__name,
res,
nc,
setup,
rsDB,
rmDB,
Granularity=self.__granularity,
SiteType=self.__siteType,
ServiceType=self.__serviceType,
ResourceType=self.__resourceType)
if 'RealBan_PolType' in policyType and self.__realBan == True:
m = Utils.voimport(actionBaseMod + ".RealBan_PolType",
self.VOExtension)
m.RealBanPolTypeActions(self.__granularity, self.__name, res,
da, csAPI, setup)
""" LogPolicyResultAction
"""
from DIRAC import S_OK, S_ERROR
from DIRAC.ResourceStatusSystem.PolicySystem.Actions.BaseAction import BaseAction
from DIRAC.ResourceStatusSystem.Utilities import Utils
ResourceManagementClient = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient"), "ResourceManagementClient"
)
class LogPolicyResultAction(BaseAction):
"""
Action that registers on the database a new entry per policy result in the
list singlePolicyResults.
"""
def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients=None):
super(LogPolicyResultAction, self).__init__(
name, decisionParams, enforcementResult, singlePolicyResults, clients
)
if clients is not None and "ResourceManagementClient" in clients:
self.rmClient = clients["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def run(self):
"""
def enforce(self, decisionParams):
""" Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
# Setup PDP with new parameters dictionary
self.pdp.setup(decisionParams)
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions['OK']:
gLogger.error(
'PEP: Something went wrong, not enforcing policies for %s' %
decisionParams)
return resDecisions
resDecisions = resDecisions['Value']
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions['decissionParams']
policyCombinedResult = resDecisions['policyCombinedResult']
singlePolicyResults = resDecisions['singlePolicyResults']
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated(decisionParams)
if not isNotUpdated['OK']:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult[
'PolicyAction']:
try:
actionMod = Utils.voimport(
'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' %
policyActionType)
except ImportError:
gLogger.error('Error importing %s action' % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
gLogger.error('Error importing %s action class' %
policyActionType)
continue
actionObj = action(policyActionName, decisionParams,
policyCombinedResult, singlePolicyResults,
self.clients)
gLogger.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult['OK']:
gLogger.error(actionResult['Message'])
return S_OK(resDecisions)
''' CacheFeederAgent This agent feeds the Cache tables with the outputs of the cache commands. ''' from DIRAC import S_OK from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.Core.LCG.GOCDBClient import GOCDBClient from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient from DIRAC.ResourceStatusSystem.Command import CommandCaller from DIRAC.ResourceStatusSystem.Utilities import Utils ResourceManagementIHEPClient = getattr( Utils.voimport( 'DIRAC.ResourceStatusSystem.Client.ResourceManagementIHEPClient' ), 'ResourceManagementIHEPClient' ) __RCSID__ = '$Id: $' AGENT_NAME = 'ResourceStatus/CacheFeederIHEPAgent' class CacheFeederIHEPAgent( AgentModule ): ''' The CacheFeederAgent feeds the cache tables for the client and the accounting. It runs periodically a set of commands, and stores it's results on the tables. ''' # Too many public methods # pylint: disable=R0904 def __init__( self, *args, **kwargs ):
PEP ( Policy Enforcement Point ) is the front-end of the whole Policy System.
Any interaction with it must go through the PEP to ensure a smooth flow.
Firstly, it loads the PDP ( Policy Decision Point ) which actually is the
module doing all dirty work ( finding policies, running them, merging their
results, etc... ). Indeed, the PEP takes the output of the PDP for a given set
of parameters ( decisionParams ) and enforces the actions that apply ( also
determined by the PDP output ).
"""
from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
from DIRAC.ResourceStatusSystem.Utilities import Utils
ResourceManagementClient = getattr( Utils.voimport( 'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient' ),'ResourceManagementClient')
ResourceStatusClient = getattr( Utils.voimport( 'DIRAC.ResourceStatusSystem.Client.ResourceStatusClient' ), 'ResourceStatusClient' )
__RCSID__ = '$Id: $'
class PEP( object ):
""" PEP ( Policy Enforcement Point )
"""
def __init__( self, clients = dict() ):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
def getPoliciesThatApply( decisionParams ):
"""
Method that sanitizes the input parameters and returns the policies that
match them. Matches the input dictionary with the policies configuration in
the CS. It returns a list of policy dictionaries that matched.
"""
decisionParams = _sanitizedecisionParams( decisionParams )
gLogger.debug("Sanitized decisionParams: %s" % str(decisionParams))
policiesThatApply = []
# Get policies configuration metadata from CS.
policiesConfig = RssConfiguration.getPolicies()
if not policiesConfig[ 'OK' ]:
return policiesConfig
policiesConfig = policiesConfig[ 'Value' ]
gLogger.debug("All policies: %s" %str(policiesConfig))
# Each policy, has the following format
# <policyName>
# \
# policyType = <policyType>
# matchParams
# \
# ...
# configParams
# \
# ...
# Get policies that match the given decisionParameters
for policyName, policySetup in policiesConfig.items():
# The parameter policyType replaces policyName, so if it is not present,
# we pick policyName
try:
policyType = policySetup[ 'policyType' ][ 0 ]
except KeyError:
policyType = policyName
#continue
# The section matchParams is not mandatory, so we set {} as default.
policyMatchParams = policySetup.get( 'matchParams', {} )
gLogger.debug("matchParams of %s: %s" %(policyName, str(policyMatchParams)))
# FIXME: make sure the values in the policyConfigParams dictionary are typed !!
policyConfigParams = {}
#policyConfigParams = policySetup.get( 'configParams', {} )
policyMatch = Utils.configMatch( decisionParams, policyMatchParams )
gLogger.debug("PolicyMatch for decisionParams %s: %s" %(decisionParams, str(policyMatch)))
policyFilter = _filterPolicies( decisionParams, policyMatchParams )
#WARNING: we need an additional filtering function when the matching
#is not straightforward (e.g. when the policy specify a 'domain', while
#the decisionParams has only the name of the element)
if policyMatch and policyFilter:
policiesThatApply.append( ( policyName, policyType, policyConfigParams ) )
gLogger.debug("policies that apply: %s" %str(policiesThatApply))
policiesToBeLoaded = []
# Gets policies parameters from code.
for policyName, policyType, _policyConfigParams in policiesThatApply:
try:
configModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.Configurations' )
policies = copy.deepcopy( configModule.POLICIESMETA )
policyMeta = policies[ policyType ]
except KeyError:
continue
# We are not going to use name / type anymore, but we keep them for debugging
# and future usage.
policyDict = { 'name' : policyName,
'type' : policyType,
'args' : {}
}
# args is one of the parameters we are going to use on the policies. We copy
# the defaults and then we update if with whatever comes from the CS.
policyDict.update( policyMeta )
policiesToBeLoaded.append( policyDict )
return S_OK( policiesToBeLoaded )
"""
Uninstallation of a DIRAC component
"""
__RCSID__ = "$Id$"
import socket
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient
from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities
from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.Core.Base import Script
from DIRAC.Core.Utilities.PromptUser import promptUser
from DIRAC import exit as DIRACexit
from DIRAC.ResourceStatusSystem.Utilities import Utils
InstallTools = getattr(Utils.voimport('DIRAC.Core.Utilities.InstallTools'),
'InstallTools')
InstallTools.exitOnError = True
force = False
def setForce(opVal):
global force
force = True
return S_OK()
Script.registerSwitch("f", "force", "Forces the removal of the logs", setForce)
Script.setUsageMessage('\n'.join([
def enforce( self, pdpIn = None, rsDBIn = None, rmDBIn = None, ncIn = None, setupIn = None,
daIn = None, csAPIIn = None, knownInfo = None ):
"""
enforce policies, using a PDP (Policy Decision Point), based on
self.__granularity (optional)
self.__name (optional)
self.__status (optional)
self.__formerStatus (optional)
self.__reason (optional)
self.__siteType (optional)
self.__serviceType (optional)
self.__realBan (optional)
self.__user (optional)
self.__futurePolicyType (optional)
self.__futureGranularity (optional)
:params:
:attr:`pdpIn`: a custom PDP object (optional)
:attr:`rsDBIn`: a custom (statuses) database object (optional)
:attr:`rmDBIn`: a custom (management) database object (optional)
:attr:`setupIn`: a string with the present setup (optional)
:attr:`ncIn`: a custom notification client object (optional)
:attr:`daIn`: a custom DiracAdmin object (optional)
:attr:`csAPIIn`: a custom CSAPI object (optional)
:attr:`knownInfo`: a string of known provided information (optional)
"""
#PDP
if pdpIn is not None:
pdp = pdpIn
else:
# Use standard DIRAC PDP
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
pdp = PDP( self.VOExtension, granularity = self.__granularity, name = self.__name,
status = self.__status, formerStatus = self.__formerStatus, reason = self.__reason,
siteType = self.__siteType, serviceType = self.__serviceType,
resourceType = self.__resourceType, useNewRes = self.useNewRes )
#DB
if rsDBIn is not None:
rsDB = rsDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
rsDB = ResourceStatusDB()
if rmDBIn is not None:
rmDB = rmDBIn
else:
# Use standard DIRAC DB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
rmDB = ResourceManagementDB()
#setup
if setupIn is not None:
setup = setupIn
else:
# get present setup
setup = CS.getSetup()[ 'Value' ]
#notification client
if ncIn is not None:
nc = ncIn
else:
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
nc = NotificationClient()
#DiracAdmin
if daIn is not None:
da = daIn
else:
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
da = DiracAdmin()
#CSAPI
if csAPIIn is not None:
csAPI = csAPIIn
else:
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
###################
# policy decision #
###################
resDecisions = pdp.takeDecision( knownInfo = knownInfo )
assert(type(resDecisions) == dict and resDecisions != {})
res = resDecisions[ 'PolicyCombinedResult' ]
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyPolTypeActions( self.__granularity, self.__name, resDecisions, res )
else:
policyType = res[ 'PolicyType' ]
if 'Resource_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Resource_PolType", self.VOExtension)
m.ResourcePolTypeActions( self.__granularity, self.__name, resDecisions, res, rsDB, rmDB )
if 'Alarm_PolType' in policyType:
m = Utils.voimport(actionBaseMod + ".Alarm_PolType", self.VOExtension)
m.AlarmPolTypeActions(self.__name, res, nc, setup, rsDB, rmDB,
Granularity=self.__granularity,
SiteType=self.__siteType,
ServiceType=self.__serviceType,
ResourceType=self.__resourceType)
if 'RealBan_PolType' in policyType and self.__realBan == True:
m = Utils.voimport(actionBaseMod + ".RealBan_PolType", self.VOExtension)
m.RealBanPolTypeActions( self.__granularity, self.__name, res, da, csAPI, setup )
def __init__( self ): configModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.Configurations' ) self.policies = copy.deepcopy( configModule.POLICIESMETA )
PEP ( Policy Enforcement Point ) is the front-end of the whole Policy System.
Any interaction with it must go through the PEP to ensure a smooth flow.
Firstly, it loads the PDP ( Policy Decision Point ) which actually is the
module doing all dirty work ( finding policies, running them, merging their
results, etc... ). Indeed, the PEP takes the output of the PDP for a given set
of parameters ( decisionParams ) and enforces the actions that apply ( also
determined by the PDP output ).
"""
from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
from DIRAC.ResourceStatusSystem.Utilities import Utils
SiteStatus = getattr(Utils.voimport('DIRAC.ResourceStatusSystem.Client.SiteStatus'), 'SiteStatus')
ResourceManagementClient = getattr(
Utils.voimport('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient'),
'ResourceManagementClient')
ResourceStatusClient = getattr(
Utils.voimport('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient'),
'ResourceStatusClient')
__RCSID__ = '$Id: $'
class PEP(object):
""" PEP ( Policy Enforcement Point )
"""
def __init__(self, clients=dict()):
def enforce(self, decisionParams):
"""Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
if not decisionParams:
self.log.warn("No decision params...?")
return S_OK()
standardParamsDict = {
"element": None,
"name": None,
"elementType": None,
"statusType": None,
"status": None,
"reason": None,
"tokenOwner": None,
# Last parameter allows policies to be de-activated
"active": "Active",
}
standardParamsDict.update(decisionParams)
if standardParamsDict["element"] is not None:
self.log = gLogger.getSubLogger("PEP/%s" % standardParamsDict["element"])
if standardParamsDict["name"] is not None:
self.log = gLogger.getSubLogger(
"PEP/%s/%s" % (standardParamsDict["element"], standardParamsDict["name"])
)
self.log.verbose(
"Enforce - statusType: %s, status: %s"
% (standardParamsDict["statusType"], standardParamsDict["status"])
)
decisionParams = dict(standardParamsDict)
# Setup PDP with new parameters dictionary
self.pdp.setup(decisionParams)
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions["OK"]:
self.log.error("Something went wrong, not enforcing policies", "%s" % decisionParams)
return resDecisions
resDecisions = resDecisions["Value"]
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions["decisionParams"]
policyCombinedResult = resDecisions["policyCombinedResult"]
singlePolicyResults = resDecisions["singlePolicyResults"]
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated(decisionParams)
if not isNotUpdated["OK"]:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult["PolicyAction"]:
try:
actionMod = Utils.voimport("DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s" % policyActionType)
except ImportError:
self.log.error("Error importing %s action" % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
self.log.error("Error importing %s action class" % policyActionType)
continue
actionObj = action(
policyActionName, decisionParams, policyCombinedResult, singlePolicyResults, self.clients
)
self.log.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult["OK"]:
self.log.error(actionResult["Message"])
return S_OK(resDecisions)
"""
__RCSID__ = '$Id$'
# pylint: disable=no-self-use
from datetime import datetime, timedelta
from types import NoneType
# DIRAC
from DIRAC import gLogger, S_OK, gConfig, S_ERROR
from DIRAC.Core.DISET.RequestHandler import RequestHandler
from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient
from DIRAC.ResourceStatusSystem.Utilities import CSHelpers, Utils
ResourceManagementClient = getattr(
Utils.voimport('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient'),
'ResourceManagementClient')
# RSS Clients
rsClient = None
rmClient = None
def initializePublisherHandler(_serviceInfo):
"""
Handler initialization in the usual horrible way.
"""
global rsClient
rsClient = ResourceStatusClient()
"""
Uninstallation of a DIRAC component
"""
__RCSID__ = "$Id$"
import socket
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient
from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities
from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.Core.Base import Script
from DIRAC.Core.Utilities.PromptUser import promptUser
from DIRAC import exit as DIRACexit
from DIRAC.ResourceStatusSystem.Utilities import Utils
InstallTools = getattr( Utils.voimport( 'DIRAC.Core.Utilities.InstallTools' ), 'InstallTools' )
InstallTools.exitOnError = True
force = False
def setForce( opVal ):
global force
force = True
return S_OK()
Script.registerSwitch( "f", "force", "Forces the removal of the logs", setForce )
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName,
'Arguments:',
' System: Name of the DIRAC system (ie: WorkloadManagement)',
''' ResourceManagementHandler
Module that allows users to access the ResourceManagementDB remotely.
'''
from DIRAC import gConfig, S_OK, gLogger
from DIRAC.Core.DISET.RequestHandler import RequestHandler
from DIRAC.ResourceStatusSystem.Utilities import Synchronizer, Utils
ResourceManagementDB = getattr(
Utils.voimport('DIRAC.ResourceStatusSystem.DB.ResourceManagementDB'),
'ResourceManagementDB')
__RCSID__ = '$Id: $'
def initializeResourceManagementHandler(_serviceInfo):
'''
Handler initialization, where we set the ResourceManagementDB as global db.
'''
global db
db = ResourceManagementDB()
# Regenerates DB tables if needed
db._checkTable()
syncObject = Synchronizer.Synchronizer()
gConfig.addListenerToNewVersionEvent(syncObject.sync)
return S_OK()
def __init__(self):
configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations")
self.C_Policies = copy.deepcopy(configModule.Policies)
def enforce( self, granularity = None, name = None, statusType = None,
status = None, formerStatus = None, reason = None,
siteType = None, serviceType = None, resourceType = None,
tokenOwner = None, useNewRes = False, knownInfo = None ):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## real ban flag #########################################################
realBan = False
if tokenOwner is not None:
if tokenOwner == 'RS_SVC':
realBan = True
## sanitize input ##########################################################
## IS IT REALLY NEEDED ??
validElements = RssConfiguration.getValidElements()
if granularity is not None and granularity not in validElements:
return S_ERROR( 'Granularity "%s" not valid' % granularity )
validStatusTypes = RssConfiguration.getValidStatusTypes()
if statusType is not None and statusType not in validStatusTypes[ granularity ]['StatusType']:
return S_ERROR( 'StatusType "%s" not valid' % statusType )
validStatus = RssConfiguration.getValidStatus()
if status is not None and status not in validStatus:
return S_ERROR( 'Status "%s" not valid' % status )
validStatus = RssConfiguration.getValidStatus()
if formerStatus is not None and formerStatus not in validStatus:
return S_ERROR( 'FormerStatus "%s" not valid' % formerStatus )
validSiteTypes = RssConfiguration.getValidSiteTypes()
if siteType is not None and siteType not in validSiteTypes:
return S_ERROR( 'SiteType "%s" not valid' % siteType )
validServiceTypes = RssConfiguration.getValidServiceTypes()
if serviceType is not None and serviceType not in validServiceTypes:
return S_ERROR( 'ServiceType "%s" not valid' % serviceType )
validResourceTypes = RssConfiguration.getValidResourceTypes()
if resourceType is not None and resourceType not in validResourceTypes:
return S_ERROR( 'ResourceType "%s" not valid' % resourceType )
## policy setup ############################################################
self.pdp.setup( granularity = granularity, name = name,
statusType = statusType, status = status,
formerStatus = formerStatus, reason = reason,
siteType = siteType, serviceType = serviceType,
resourceType = resourceType, useNewRes = useNewRes )
## policy decision #########################################################
resDecisions = self.pdp.takeDecision( knownInfo = knownInfo )
## record all results before doing anything else
for resP in resDecisions[ 'SinglePolicyResults' ]:
if not resP.has_key( 'OLD' ):
self.clients[ "rmClient" ].insertPolicyResultLog( granularity, name,
resP[ 'PolicyName' ],
statusType,
resP[ 'Status' ],
resP[ 'Reason' ], now )
else:
gLogger.warn( 'OLD: %s' % resP )
res = resDecisions[ 'PolicyCombinedResult' ]
actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"
# Security mechanism in case there is no PolicyType returned
if res == {}:
EmptyAction(granularity, name, statusType, resDecisions).run()
else:
policyType = res[ 'PolicyType' ]
if 'Resource_PolType' in policyType:
action = Utils.voimport( '%s.ResourceAction' % actionBaseMod )
action.ResourceAction(granularity, name, statusType, resDecisions,
rsClient=self.rsClient,
rmClient=self.rmClient).run()
if 'Alarm_PolType' in policyType:
action = Utils.voimport( '%s.AlarmAction' % actionBaseMod )
action.AlarmAction(granularity, name, statusType, resDecisions,
Clients=self.clients,
Params={"Granularity" : granularity,
"SiteType" : siteType,
"ServiceType" : serviceType,
"ResourceType" : resourceType}).run()
if 'RealBan_PolType' in policyType and realBan:
action = Utils.voimport( '%s.RealBanAction' % actionBaseMod )
action.RealBanAction(granularity, name, resDecisions).run()
return resDecisions
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def enforce( self, decisionParams ):
""" Given a dictionary with decisionParams, it is passed to the PDP, which
will return ( in case there is a/are positive match/es ) a dictionary containing
three key-pair values: the original decisionParams ( `decisionParams` ), all
the policies evaluated ( `singlePolicyResults` ) and the computed final result
( `policyCombinedResult` ).
To know more about decisionParams, please read PDP.setup where the decisionParams
are sanitized.
examples:
>>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
>>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
:Parameters:
**decisionParams** - `dict`
dictionary with the parameters that will be used to match policies.
"""
# Setup PDP with new parameters dictionary
self.pdp.setup( decisionParams )
# Run policies, get decision, get actions to apply
resDecisions = self.pdp.takeDecision()
if not resDecisions[ 'OK' ]:
gLogger.error( 'PEP: Something went wrong, not enforcing policies for %s' % decisionParams )
return resDecisions
resDecisions = resDecisions[ 'Value' ]
# We take from PDP the decision parameters used to find the policies
decisionParams = resDecisions[ 'decissionParams' ]
policyCombinedResult = resDecisions[ 'policyCombinedResult' ]
singlePolicyResults = resDecisions[ 'singlePolicyResults' ]
# We have run the actions and at this point, we are about to execute the actions.
# One more final check before proceeding
isNotUpdated = self.__isNotUpdated( decisionParams )
if not isNotUpdated[ 'OK' ]:
return isNotUpdated
for policyActionName, policyActionType in policyCombinedResult[ 'PolicyAction' ]:
try:
actionMod = Utils.voimport( 'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' % policyActionType )
except ImportError:
gLogger.error( 'Error importing %s action' % policyActionType )
continue
try:
action = getattr( actionMod, policyActionType )
except AttributeError:
gLogger.error( 'Error importing %s action class' % policyActionType )
continue
actionObj = action( policyActionName, decisionParams, policyCombinedResult,
singlePolicyResults, self.clients )
gLogger.debug( ( policyActionName, policyActionType ) )
actionResult = actionObj.run()
if not actionResult[ 'OK' ]:
gLogger.error( actionResult[ 'Message' ] )
return S_OK( resDecisions )
