def _getStatus(self, name, panel):
#get RSS status
RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]
#get DIRAC status
if panel in ('Site_Panel', 'SE_Panel'):
if panel == 'Site_Panel':
DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
if DIRACStatus['OK']:
DIRACStatus = DIRACStatus['Value'][name].pop()[0]
else:
raise RSSException, Utils.where(self, self._getStatus)
elif panel == 'SE_Panel':
ra = getStorageElementStatus(name, 'ReadAccess')['Value']
wa = getStorageElementStatus(name, 'WriteAccess')['Value']
DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}
status = {
name: {
'RSSStatus': RSSStatus,
'DIRACStatus': DIRACStatus
}
}
else:
status = {name: {'RSSStatus': RSSStatus}}
return status
def getValue( val, default ):
'''Wrapper around gConfig.getValue. Returns typed values'''
res = gConfig.getValue( val, default )
if Utils.isiterable( res ):
return [ Utils.typedobj_of_string(e) for e in res ]
else:
return Utils.typedobj_of_string( res )
def _getStatus(self, name, panel):
#get RSS status
RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]
#get DIRAC status
if panel in ('Site_Panel', 'SE_Panel'):
if panel == 'Site_Panel':
DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
if DIRACStatus['OK']:
DIRACStatus = DIRACStatus['Value'][name].pop()[0]
else:
raise RSSException, Utils.where(self, self._getStatus)
elif panel == 'SE_Panel':
ra = getStorageElementStatus(name, 'ReadAccess')['Value']
wa = getStorageElementStatus(name, 'WriteAccess')['Value']
DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}
status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } }
else:
status = { name : { 'RSSStatus': RSSStatus} }
return status
def getValue(v):
"""Wrapper around gConfig.getValue. Returns typed values instead of
a string value"""
res = gConfig.getValue(v)
if res.find(",") > -1: # res is a list of values
return [Utils.typedobj_of_string(e) for e in List.fromChar(res)]
else: return Utils.typedobj_of_string(res)
def getValue(val, default):
'''Wrapper around gConfig.getValue. Returns typed values'''
res = gConfig.getValue(val, default)
if Utils.isiterable(res):
return [Utils.typedobj_of_string(e) for e in res]
else:
return Utils.typedobj_of_string(res)
def typed_dict_of_dict(d):
for k in d:
if type(d[k]) == dict:
d[k] = typed_dict_of_dict(d[k])
else:
if d[k].find(",") > -1:
d[k] = [Utils.typedobj_of_string(e) for e in List.fromChar(d[k])]
else:
d[k] = Utils.typedobj_of_string(d[k])
return d
def __syncNode(self,
NodeInCS,
resourcesInDB,
resourceType,
serviceType,
site="NULL"):
nodesToUpdate = NodeInCS - resourcesInDB
if len(nodesToUpdate) > 0:
gLogger.debug(str(NodeInCS))
gLogger.debug(str(nodesToUpdate))
# Update Service table
siteInGOCDB = [
self.__getServiceEndpointInfo(node) for node in nodesToUpdate
]
siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
#sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
sites = []
for sInGOCDB in siteInGOCDB:
siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME'])
if not siteName['OK']:
gLogger.error(siteName['Message'])
return siteName
sites.append(siteName['Value'])
sites = Utils.list_sanitize(Utils.list_flatten(sites))
_ = [self.__updateService(s, serviceType) for s in sites]
# Update Resource table
for node in NodeInCS:
if serviceType == "Computing":
resourceType = CS.getCEType(site, node)
if node not in resourcesInDB and node is not None:
try:
siteInGOCDB = self.__getServiceEndpointInfo(
node)[0]['SITENAME']
except IndexError: # No INFO in GOCDB: Node does not exist
gLogger.warn(
"Node %s is not in GOCDB!! Considering that it does not exists!"
% node)
continue
assert (type(siteInGOCDB) == str)
#Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
res = self.rsClient.addOrModifyResource(
node, resourceType, serviceType, site, siteInGOCDB)
if not res['OK']:
gLogger.error(res['Message'])
return res
resourcesInDB.add(node)
def __init__(self):
self.log = gLogger.getSubLogger('PilotsLoggingDB')
result = getDBParameters('WorkloadManagement/PilotsLoggingDB')
if not result['OK']:
raise RuntimeError('Cannot get database parameters: %s' %
result['Message'])
dbParameters = result['Value']
self.dbHost = dbParameters['Host']
self.dbPort = dbParameters['Port']
self.dbUser = dbParameters['User']
self.dbPass = dbParameters['Password']
self.dbName = dbParameters['DBName']
# These are the list of tables that will be created.
# They can be extended in an extension module
self.tablesList = getattr(
Utils.voimport(
'DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB'),
'TABLESLIST')
self.__initializeConnection()
resp = self.__initializeDB()
if not resp['OK']:
raise Exception("Couldn't create tables: " + resp['Message'])
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def initializeResourceStatusHandler(_serviceInfo):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
# Publisher is on boxes right now
#
# rmDB = ResourceStatusDB()
# cc = CommandCaller()
# global VOExtension
# VOExtension = getExt()
# ig = InfoGetter( VOExtension )
# WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
# global publisher
# publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
# infoGetterIn = ig, WMSAdminIn = WMSAdmin )
syncModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Utilities.Synchronizer')
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent(syncObject.sync)
return S_OK()
def __init__(self, parentLogger=None):
if not parentLogger:
parentLogger = gLogger
self.log = parentLogger.getSubLogger(self.__class__.__name__)
result = getDBParameters("WorkloadManagement/PilotsLoggingDB")
if not result["OK"]:
raise RuntimeError("Cannot get database parameters: %s" % result["Message"])
dbParameters = result["Value"]
self.dbHost = dbParameters["Host"]
self.dbPort = dbParameters["Port"]
self.dbUser = dbParameters["User"]
self.dbPass = dbParameters["Password"]
self.dbName = dbParameters["DBName"]
# These are the list of tables that will be created.
# They can be extended in an extension module
self.tablesList = getattr(Utils.voimport("DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB"), "TABLESLIST")
self.__initializeConnection()
resp = self.__initializeDB()
if not resp["OK"]:
raise Exception("Couldn't create tables: " + resp["Message"])
def getValidStatusTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Resources
'''
DEFAULTS = {
'Site' : { 'StatusType' : "''" },
'Service' : { 'StatusType' : "''" },
'Resource' : { 'StatusType' : "''" },
'StorageElement': { 'StatusType' : [ 'Read', 'Write', 'Remove', 'Check' ] }
}
opHelper = Operations()
sections = opHelper.getSections( 'RSSConfiguration/GeneralConfig/Resources' )
if not sections[ 'OK' ]:
return DEFAULTS
result = {}
for section in sections[ 'Value' ]:
res = opHelper.getValue( 'RSSConfiguration/GeneralConfig/Resources/%s/StatusType' % section )
if res is None:
if DEFAULTS.has_key( section ):
result[ section ] = { 'StatusType' : DEFAULTS[ section ] }
else:
result[ section ] = { 'StatusType' : None }
else:
result[ section ] = { 'StatusType' : Utils.getTypedList( res ) }
return result
def __getPolTypes(self, granularity, status=None, formerStatus=None, newStatus=None,
siteType=None, serviceType=None, resourceType=None):
"""Get Policy Types from config that match the given keyword
arguments"""
# This dict is constructed to be used with function dictMatch that
# helps selecting policies. **kwargs are not used due to the fact
# that it's too dangerous here.
argsdict = {'Granularity': granularity,
'Status': status,
'FormerStatus': formerStatus,
'NewStatus': newStatus,
'SiteType': siteType,
'ServiceType': serviceType,
'ResourceType': resourceType}
pTconfig = getTypedDictRootedAt("PolicyTypes")
pTypes = []
for pt in pTconfig:
if Utils.dictMatch(argsdict, pTconfig[pt]):
pTypes.append(pt)
for pt_name in pTypes:
if 'Alarm_PolType' in pt_name:
pTypes.remove(pt_name)
pTypes.append('Alarm_PolType')
return pTypes
def initializeResourceStatusHandler( _serviceInfo ):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
# Publisher is on boxes right now
#
# rmDB = ResourceStatusDB()
# cc = CommandCaller()
# global VOExtension
# VOExtension = getExt()
# ig = InfoGetter( VOExtension )
# WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
# global publisher
# publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
# infoGetterIn = ig, WMSAdminIn = WMSAdmin )
syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer' )
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent( syncObject.sync )
return S_OK()
def getSpaceTokenEndpoints():
''' Get Space Token Endpoints '''
return Utils.getCSTree('Shares/Disk')
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __getPanelsInfo( self, granularity, statusType = None, status = None,
formerStatus = None, siteType = None, serviceType = None,
resourceType = None, panel_name = None, useNewRes = False ):
info = []
# First, select only policies we want.
argsdict = {'Granularity' : granularity,
'StatusType' : statusType,
'Status' : status,
'FormerStatus' : formerStatus,
'SiteType' : siteType,
'ServiceType' : serviceType,
'ResourceType' : resourceType}
all_policies = getTypedDictRootedAtOperations("Policies")
selected_policies = []
for p in all_policies:
if Utils.dictMatch(argsdict, all_policies[p]):
selected_policies.append(p)
for p in selected_policies: # For selected policies
if panel_name in self.C_Policies[p].keys(): # For selected panel_name (arguments)
toAppend = copy.deepcopy(self.C_Policies[p][panel_name]) # type(toAppend) = list
# Put CommandIn and args to correct values according to useNewRes
if useNewRes:
for panel in toAppend:
for info_type in panel.keys():
if type(panel[info_type]) == dict:
try:
panel[info_type]['CommandIn'] = panel[info_type]['CommandInNewRes']
del panel[info_type]['CommandInNewRes']
except KeyError:
pass
try:
panel[info_type]['args'] = panel[info_type]['argsNewRes']
del panel[info_type]['argsNewRes']
except KeyError:
pass
else:
for panel in toAppend:
for info_type in panel.keys():
try:
del panel[info_type]['CommandInNewRes']
except KeyError:
pass
try:
del panel[info_type]['argsNewRes']
except KeyError:
pass
info.append({p:toAppend})
return info
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( 'ResourceStatusDB' )
#These are the list of tables that will be created.
#They can be extended in an extension module
self.tablesList = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceStatusDB' ),
'TABLESLIST')
self.tablesListWithID = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceStatusDB' ),
'TABLESLISTWITHID')
self.extensions = gConfig.getValue( 'DIRAC/Extensions', [] )
self.__initializeConnection( 'ResourceStatus/ResourceStatusDB' )
self.__initializeDB()
def enforce(self, decissionParams):
'''
Enforce policies for given set of keyworkds. To be better explained.
'''
## policy decision point setup #############################################
self.pdp.setup(decissionParams)
## policy decision #########################################################
resDecisions = self.pdp.takeDecision()
if not resDecisions['OK']:
gLogger.error(
'PEP: Something went wrong, not enforcing policies for %s' %
decissionParams)
return resDecisions
resDecisions = resDecisions['Value']
# We take from PDP the decision parameters used to find the policies
decissionParams = resDecisions['decissionParams']
policyCombinedResult = resDecisions['policyCombinedResult']
singlePolicyResults = resDecisions['singlePolicyResults']
for policyActionName, policyActionType in policyCombinedResult[
'PolicyAction']:
try:
actionMod = Utils.voimport(
'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' %
policyActionType)
except ImportError:
gLogger.error('Error importing %s action' % policyActionType)
continue
try:
action = getattr(actionMod, policyActionType)
except AttributeError:
gLogger.error('Error importing %s action class' %
policyActionType)
continue
actionObj = action(policyActionName, decissionParams,
policyCombinedResult, singlePolicyResults,
self.clients)
gLogger.debug((policyActionName, policyActionType))
actionResult = actionObj.run()
if not actionResult['OK']:
gLogger.error(actionResult['Message'])
return S_OK(resDecisions)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def policyInvocation( self, granularity = None, name = None,
status = None, policy = None, args = None, pName = None,
pModule = None, extraArgs = None, commandIn = None ):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if not policy:
try:
policyModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.%s' % pModule )
except ImportError:
_msg = 'Unable to import a policy module named %s, falling back on AlwaysFalse_Policy.' % pModule
gLogger.warn( _msg )
policyModule = __import__( 'DIRAC.ResourceStatusSystem.Policy.AlwaysFalse_Policy',
globals(), locals(), ['*'] )
pModule = 'AlwaysFalse_Policy'
try:
policy = getattr( policyModule, pModule )()
except AttributeError as exc:
print policyModule, pModule
raise exc
if not args:
args = ( granularity, name )
if extraArgs:
args = args + tuple( extraArgs )
if commandIn:
commandIn = self.cCaller.setCommandObject( commandIn )
for clientName, clientInstance in self.clients.items():
self.cCaller.setAPI( commandIn, clientName, clientInstance )
res = self._innerEval( policy, args, commandIn = commandIn )
# Just adding the PolicyName to the result of the evaluation of the policy
res[ 'PolicyName' ] = pName
return res
def __init__(self, VOExtension):
"""
Standard constructor
:params:
:attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
"""
configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
self.C_Policies = copy.deepcopy(configModule.Policies)
def _getUsersToNotify(self):
groups = CS.getTypedDictRootedAtOperations("AssigneeGroups/" +
CS.getSetup()).values()
concerned_groups = [
g for g in groups if Utils.dictMatch(self.kw["Params"], g)
]
return [{
'Users': g['Users'],
'Notifications': g['Notifications']
} for g in concerned_groups]
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
WMSAdminIn = None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool( 2, 5 )
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
def policyInvocation(self, decisionParams, policyDict):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if 'module' not in policyDict:
return S_ERROR('Malformed policyDict %s' % policyDict)
pModuleName = policyDict['module']
if 'command' not in policyDict:
return S_ERROR('Malformed policyDict %s' % policyDict)
pCommand = policyDict['command']
if 'args' not in policyDict:
return S_ERROR('Malformed policyDict %s' % policyDict)
pArgs = policyDict['args']
try:
policyModule = Utils.voimport(
'DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName)
except ImportError:
return S_ERROR(
'Unable to import DIRAC.ResourceStatusSystem.Policy.%s' %
pModuleName)
if not hasattr(policyModule, pModuleName):
return S_ERROR('%s has no attibute %s' %
(policyModule, pModuleName))
policy = getattr(policyModule, pModuleName)()
command = self.cCaller.commandInvocation(pCommand, pArgs,
decisionParams, self.clients)
if not command['OK']:
return command
command = command['Value']
evaluationResult = self.policyEvaluation(policy, command)
if evaluationResult['OK']:
evaluationResult['Value']['Policy'] = policyDict
return evaluationResult
def getValidPolicyTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/PolicyTypes
'''
DEFAULTS = [ 'Resource_PolType', 'Alarm_PolType', 'Collective_PolType', 'RealBan_PolType' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/PolicyTypes' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def __init__(self, VOExtension):
"""
Standard constructor
:params:
:attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
"""
configModule = Utils.voimport(
"DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
self.C_Policies = copy.deepcopy(configModule.Policies)
def __init__( self ):
""" Constructor. Imports the policy configurations containing the command
information, among other things.
examples:
>>> iGetter = InfoGetter()
"""
configModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.Configurations' )
self.policies = copy.deepcopy( configModule.POLICIESMETA )
def getValidServiceTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/ServiceType
'''
DEFAULTS = [ 'Computing', 'Storage', 'VO-BOX', 'VOMS', 'CondDB' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/ServiceType' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidPolicyResult():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/PolicyResult
'''
DEFAULTS = [ 'Error', 'Unknown', 'Banned', 'Probing', 'Bad', 'Active' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/PolicyResult' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidElements():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Granularity
'''
DEFAULTS = [ 'Site', 'Service', 'Resource', 'StorageElement' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/Granularity' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidStatus():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Status
'''
DEFAULTS = [ 'Active', 'Bad', 'Probing', 'Banned' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/Status' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidResourceTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/ResourceType
'''
DEFAULTS = [ 'CE', 'CREAMCE', 'SE', 'LFC_C', 'LFC_L', 'FTS', 'VOMS' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/ResourceType' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def getValidSiteTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/SiteType
'''
DEFAULTS = [ 'T1', 'T2', 'T3', 'T4' ]
result = Operations().getValue( 'RSSConfiguration/GeneralConfig/SiteType' )
if result is not None:
return Utils.getTypedList( result )
return DEFAULTS
def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"):
nodesToUpdate = NodeInCS - resourcesInDB
if len(nodesToUpdate) > 0:
gLogger.debug(str(NodeInCS))
gLogger.debug(str(nodesToUpdate))
# Update Service table
siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate]
siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
#sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
sites = []
for sInGOCDB in siteInGOCDB:
siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] )
if not siteName[ 'OK' ]:
gLogger.error( siteName[ 'Message' ] )
return siteName
sites.append( siteName[ 'Value' ] )
sites = Utils.list_sanitize( Utils.list_flatten( sites ) )
_ = [ self.__updateService(s, serviceType) for s in sites ]
# Update Resource table
for node in NodeInCS:
if serviceType == "Computing":
resourceType = CS.getCEType(site, node)
if node not in resourcesInDB and node is not None:
try:
siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME']
except IndexError: # No INFO in GOCDB: Node does not exist
gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node)
continue
assert(type(siteInGOCDB) == str)
#Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
resourcesInDB.add( node )
def __init__( self, VOExtension, granularity = None, name = None, status = None, formerStatus = None,
reason = None, siteType = None, serviceType = None, resourceType = None,
tokenOwner = None, useNewRes = False ):
self.VOExtension = VOExtension
try:
self.__granularity = Utils.assignOrRaise( granularity, ValidRes, InvalidRes, self, self.__init__ )
except NameError:
pass
self.__name = name
self.__status = Utils.assignOrRaise( status, ValidStatus, InvalidStatus, self, self.__init__ )
self.__formerStatus = Utils.assignOrRaise( formerStatus, ValidStatus, InvalidStatus, self, self.__init__ )
self.__reason = reason
self.__siteType = Utils.assignOrRaise( siteType, ValidSiteType, InvalidSiteType, self, self.__init__ )
self.__serviceType = Utils.assignOrRaise( serviceType, ValidServiceType, InvalidServiceType, self, self.__init__ )
self.__resourceType = Utils.assignOrRaise( resourceType, ValidResourceType, InvalidResourceType, self, self.__init__ )
self.__realBan = False
if tokenOwner is not None:
if tokenOwner == 'RS_SVC':
self.__realBan = True
self.useNewRes = useNewRes
def getTypedDictRootedAt( path ):
retval = {}
opts = gConfig.getOptionsDict( path )
secs = gConfig.getSections( path )
if not opts[ 'OK' ]:
raise CSError, opts[ 'Message' ]
if not secs[ 'OK' ]:
raise CSError, secs[ 'Message' ]
opts = opts[ 'Value' ]
secs = secs[ 'Value' ]
for k in opts:
if opts[ k ].find( "," ) > -1:
retval[ k ] = [ Utils.typedobj_of_string(e) for e in List.fromChar(opts[k]) ]
else:
retval[ k ] = Utils.typedobj_of_string( opts[ k ] )
for i in secs:
retval[ i ] = getTypedDictRootedAt( path + "/" + i )
return retval
def __init__(self):
"""c'tor
:param self: self reference
"""
super(ResourceStatusDB, self).__init__()
# These are the list of tables that will be created.
# They can be extended in an extension module
self.tablesList = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"),
"TABLESLIST")
self.tablesListWithID = getattr(
Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"),
"TABLESLISTWITHID")
self.extensions = gConfig.getValue("DIRAC/Extensions", [])
self._initializeConnection("ResourceStatus/ResourceStatusDB")
# Create required tables
self._createTablesIfNotThere(self.tablesList)
self._createTablesIfNotThere(self.tablesListWithID)
def policyInvocation( self, decisionParams, policyDict ):
'''
Invokes a policy:
1. If :attr:`policy` is None, import the policy module specified
with :attr:`pModule` (e.g. 'DT_Policy').
1.1. Create a policy object.
2. Set the policy arguments (usually :attr:`granularity`,
:attr:`name`) + :attr:`extraArgs`.
3. If commandIn is specified (normally it is), use
:meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
to get a command object
'''
if not 'module' in policyDict:
return S_ERROR( 'Malformed policyDict %s' % policyDict )
pModuleName = policyDict[ 'module' ]
if not 'command' in policyDict:
return S_ERROR( 'Malformed policyDict %s' % policyDict )
pCommand = policyDict[ 'command' ]
if not 'args' in policyDict:
return S_ERROR( 'Malformed policyDict %s' % policyDict )
pArgs = policyDict[ 'args' ]
try:
policyModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName )
except ImportError:
return S_ERROR( 'Unable to import DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName )
if not hasattr( policyModule, pModuleName ):
return S_ERROR( '%s has no attibute %s' % ( policyModule, pModuleName ) )
policy = getattr( policyModule, pModuleName )()
command = self.cCaller.commandInvocation( pCommand, pArgs, decisionParams, self.clients )
if not command[ 'OK' ]:
return command
command = command[ 'Value' ]
evaluationResult = self.policyEvaluation( policy, command )
if evaluationResult[ 'OK' ]:
evaluationResult[ 'Value' ][ 'Policy' ] = policyDict
return evaluationResult
def initializeResourceStatusHandler( _serviceInfo ):
'''
Handler initialization, where we set the ResourceStatusDB as global db, and
we instantiate the synchronizer.
'''
global db
db = ResourceStatusDB()
syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer' )
syncObject = syncModule.Synchronizer()
gConfig.addListenerToNewVersionEvent( syncObject.sync )
return S_OK()
def __getPolToEval(self, granularity, status=None, formerStatus=None, siteType=None,
serviceType=None, resourceType=None, useNewRes=False):
# This dict is constructed to be used with function dictMatch that
# helps selecting policies. **kwargs are not used due to the fact
# that it's too dangerous here.
argsdict = {'Granularity': granularity,
'Status': status,
'FormerStatus': formerStatus,
'SiteType': siteType,
'ServiceType': serviceType,
'ResourceType': resourceType}
pConfig = getTypedDictRootedAt("Policies")
pol_to_eval = []
for p in pConfig:
if Utils.dictMatch(argsdict, pConfig[p]):
pol_to_eval.append(p)
polToEval_Args = []
for p in pol_to_eval:
try:
moduleName = self.C_Policies[p]['module']
except KeyError:
moduleName = None
try:
ConfirmationPolicy = self.C_Policies[p]['ConfirmationPolicy']
except KeyError:
ConfirmationPolicy = None
if useNewRes:
try:
commandIn = self.C_Policies[p]['commandInNewRes']
except KeyError:
commandIn = self.C_Policies[p]['commandIn']
try:
args = self.C_Policies[p]['argsNewRes']
except KeyError:
args = self.C_Policies[p]['args']
else:
commandIn = self.C_Policies[p]['commandIn']
args = self.C_Policies[p]['args']
polToEval_Args.append({'Name' : p, 'Module' : moduleName, 'args' : args,
'ConfirmationPolicy' : ConfirmationPolicy,
'commandIn' : commandIn})
return polToEval_Args
def __getPolToEval( self, granularity, statusType = None, status=None,
formerStatus=None, siteType=None, serviceType=None,
resourceType=None, useNewRes=False ):
"""Returns a possibly empty list of dicts, each dict containing
enough information to evaluate a given policy"""
# This dict is constructed to be used with function dictMatch that
# helps selecting policies. **kwargs are not used due to the fact
# that it's too dangerous here.
argsdict = { 'Granularity' : granularity,
'StatusType' : statusType,
'Status' : status,
'FormerStatus' : formerStatus,
'SiteType' : siteType,
'ServiceType' : serviceType,
'ResourceType' : resourceType }
pConfig = getTypedDictRootedAtOperations("Policies")
pol_to_eval = (p for p in pConfig if Utils.dictMatch(argsdict, pConfig[p]))
polToEval_Args = []
for p in pol_to_eval:
try:
moduleName = self.C_Policies[p]['module']
except KeyError:
moduleName = None
try:
ConfirmationPolicy = self.C_Policies[p]['ConfirmationPolicy']
except KeyError:
ConfirmationPolicy = None
if useNewRes:
try:
commandIn = self.C_Policies[p]['commandInNewRes']
except KeyError:
commandIn = self.C_Policies[p]['commandIn']
try:
args = self.C_Policies[p]['argsNewRes']
except KeyError:
args = self.C_Policies[p]['args']
else:
commandIn = self.C_Policies[p]['commandIn']
args = self.C_Policies[p]['args']
polToEval_Args.append({'Name' : p, 'Module' : moduleName, 'args' : args,
'ConfirmationPolicy' : ConfirmationPolicy,
'commandIn' : commandIn})
return polToEval_Args
def getTypedDictRootedAt(path):
retval = {}
opts = gConfig.getOptionsDict(path)
secs = gConfig.getSections(path)
if not opts['OK']:
raise CSError, opts['Message']
if not secs['OK']:
raise CSError, secs['Message']
opts = opts['Value']
secs = secs['Value']
for k in opts:
if opts[k].find(",") > -1:
retval[k] = [
Utils.typedobj_of_string(e) for e in List.fromChar(opts[k])
]
else:
retval[k] = Utils.typedobj_of_string(opts[k])
for i in secs:
retval[i] = getTypedDictRootedAt(path + "/" + i)
return retval
def __init__(self):
"""c'tor
:param self: self reference
"""
super(ResourceManagementDB, self).__init__()
# This is the list of tables that will be created.
# It can be extended in an extension module
self.tablesList = getattr(Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceManagementDB"), "TABLESLIST")
self._initializeConnection("ResourceStatus/ResourceManagementDB")
# Create required tables
self._createTablesIfNotThere(self.tablesList)
def __loadTestObj(self):
_module_pre = 'DIRAC.ResourceStatusSystem.SAM.SAMTest.'
for testType, testDict in self.tests.items():
moduleName = testDict[ 'module' ]
args = testDict.get( 'args', {} )
args.update( testDict[ 'match' ] )
args[ 'TestType' ] = testType
try:
testModule = Utils.voimport( _module_pre + moduleName )
except ImportError, e:
gLogger.error( "Unable to import %s, %s" % ( _module_pre + moduleName, e ) )
continue
testClass = getattr( testModule, moduleName )
obj = testClass(args, self.apis)
testDict[ 'object' ] = obj
def __loadTestObj(self):
_module_pre = 'DIRAC.ResourceStatusSystem.SAM.SAMTest.'
for testType, testDict in self.tests.items():
moduleName = testDict['module']
args = testDict.get('args', {})
args.update(testDict['match'])
args['TestType'] = testType
try:
testModule = Utils.voimport(_module_pre + moduleName)
except ImportError, e:
gLogger.error("Unable to import %s, %s" %
(_module_pre + moduleName, e))
continue
testClass = getattr(testModule, moduleName)
obj = testClass(args, self.apis)
testDict['object'] = obj
def commandInvocation(commandTuple,
pArgs=None,
decissionParams=None,
clients=None):
'''
Returns a command object, given commandTuple
:params:
`commandTuple`: a tuple, where commandTuple[0] is a module name and
commandTuple[1] is a class name (inside the module)
'''
if commandTuple is None:
return S_OK(None)
# decission params can be a dictionary passed with all the element parameters
# used mostly by the PDP to inject all relevant information
if decissionParams is None:
decissionParams = {}
# arguments hardcoded on Configurations.py for the policy
if pArgs is None:
pArgs = {}
try:
cModule = commandTuple[0]
cClass = commandTuple[1]
commandModule = Utils.voimport('DIRAC.ResourceStatusSystem.Command.' +
cModule)
except ImportError:
return S_ERROR("Import error for command %s." % (cModule))
if not hasattr(commandModule, cClass):
return S_ERROR('%s has no %s' % (cModule, cClass))
# We merge decision parameters and policy arguments.
newArgs = copy.deepcopy(decissionParams)
newArgs.update(pArgs)
commandObject = getattr(commandModule, cClass)(newArgs, clients)
return S_OK(commandObject)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def setCommandObject( self, comm ):
"""
Returns a command object, given comm
:params:
`comm`: a tuple, where comm[0] is a module name and comm[1] is a class name (inside the module)
"""
try:
cModule = comm[0]
cClass = comm[1]
commandModule = Utils.voimport("DIRAC.ResourceStatusSystem.Command." + cModule)
except ImportError:
gLogger.warn("Command %s/%s not found, using dummy command DoNothing_Command." % (cModule, cClass))
cClass = "DoNothing_Command"
commandModule = __import__("DIRAC.ResourceStatusSystem.Command.DoNothing_Command", globals(), locals(), ['*'])
c = getattr(commandModule, cClass)()
return c
def getUsersToNotify(setup, kwargs):
"""Get a list of users to notify (helper function for AlarmPolTypeActions)
Optional keyword arguments:
- Granularity
- SiteType
- ServiceType
- ResourceType
"""
notifications = []
groups = CS.getTypedDictRootedAt("AssigneeGroups/" + setup)
for k in groups:
if Utils.dictMatch(kwargs, groups[k]):
notifications.append({
'Users': groups[k]['Users'],
'Notifications': groups[k]['Notifications']
})
return notifications
def __getServiceEndpointInfo(self, node):
#res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) )
res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node )
if res['OK']:
res = res[ 'Value' ]
else:
gLogger.warn( 'Error getting hostname info for %s' % node )
return []
if res == []:
#res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) )
url = Utils.canonicalURL(node)
res = self.GOCDBClient.getServiceEndpointInfo('hostname', url )
if res['OK']:
res = res[ 'Value' ]
else:
gLogger.warn( 'Error getting canonical hostname info for %s' % node )
res = []
return res
def __getPolTypes( self, granularity, statusType=None, status=None,
formerStatus=None, newStatus=None, siteType=None,
serviceType=None, resourceType=None ):
"""Get Policy Types from config that match the given keyword
arguments. Always returns a generator object, possibly empty."""
# This dict is constructed to be used with function dictMatch that
# helps selecting policies. **kwargs are not used due to the fact
# that it's too dangerous here.
argsdict = {'Granularity' : granularity,
'StatusType' : statusType,
'Status' : status,
'FormerStatus' : formerStatus,
'NewStatus' : newStatus,
'SiteType' : siteType,
'ServiceType' : serviceType,
'ResourceType' : resourceType }
pTconfig = RssConfiguration.getValidPolicyTypes()
return (pt for pt in pTconfig if Utils.dictMatch(argsdict, pTconfig[pt]))
