Example #1
0
    def _getStatus(self, name, panel):

        #get RSS status
        RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]

        #get DIRAC status
        if panel in ('Site_Panel', 'SE_Panel'):

            if panel == 'Site_Panel':
                DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
                if DIRACStatus['OK']:
                    DIRACStatus = DIRACStatus['Value'][name].pop()[0]
                else:
                    raise RSSException, Utils.where(self, self._getStatus)

            elif panel == 'SE_Panel':
                ra = getStorageElementStatus(name, 'ReadAccess')['Value']
                wa = getStorageElementStatus(name, 'WriteAccess')['Value']
                DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}

            status = {
                name: {
                    'RSSStatus': RSSStatus,
                    'DIRACStatus': DIRACStatus
                }
            }

        else:
            status = {name: {'RSSStatus': RSSStatus}}

        return status
Example #2
0
File: CS.py Project: bmb/DIRAC
def getValue( val, default ):
  '''Wrapper around gConfig.getValue. Returns typed values'''
  res = gConfig.getValue( val, default )
  if Utils.isiterable( res ):
    return [ Utils.typedobj_of_string(e) for e in res ]
  else:
    return Utils.typedobj_of_string( res )
Example #3
0
  def _getStatus(self, name, panel):

    #get RSS status
    RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]

    #get DIRAC status
    if panel in ('Site_Panel', 'SE_Panel'):

      if panel == 'Site_Panel':
        DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
        if DIRACStatus['OK']:
          DIRACStatus = DIRACStatus['Value'][name].pop()[0]
        else:
          raise RSSException, Utils.where(self, self._getStatus)

      elif panel == 'SE_Panel':
        ra = getStorageElementStatus(name, 'ReadAccess')['Value']
        wa = getStorageElementStatus(name, 'WriteAccess')['Value']
        DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}

      status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } }

    else:
      status = { name : { 'RSSStatus': RSSStatus} }


    return status
Example #4
0
def getValue(v):
  """Wrapper around gConfig.getValue. Returns typed values instead of
  a string value"""
  res = gConfig.getValue(v)
  if res.find(",") > -1: # res is a list of values
    return [Utils.typedobj_of_string(e) for e in List.fromChar(res)]
  else: return Utils.typedobj_of_string(res)
Example #5
0
def getValue(val, default):
    '''Wrapper around gConfig.getValue. Returns typed values'''
    res = gConfig.getValue(val, default)
    if Utils.isiterable(res):
        return [Utils.typedobj_of_string(e) for e in res]
    else:
        return Utils.typedobj_of_string(res)
Example #6
0
 def typed_dict_of_dict(d):
   for k in d:
     if type(d[k]) == dict:
       d[k] = typed_dict_of_dict(d[k])
     else:
       if d[k].find(",") > -1:
         d[k] = [Utils.typedobj_of_string(e) for e in List.fromChar(d[k])]
       else:
         d[k] = Utils.typedobj_of_string(d[k])
   return d
Example #7
0
    def __syncNode(self,
                   NodeInCS,
                   resourcesInDB,
                   resourceType,
                   serviceType,
                   site="NULL"):

        nodesToUpdate = NodeInCS - resourcesInDB
        if len(nodesToUpdate) > 0:
            gLogger.debug(str(NodeInCS))
            gLogger.debug(str(nodesToUpdate))

        # Update Service table
        siteInGOCDB = [
            self.__getServiceEndpointInfo(node) for node in nodesToUpdate
        ]
        siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
        #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]

        sites = []
        for sInGOCDB in siteInGOCDB:
            siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME'])
            if not siteName['OK']:
                gLogger.error(siteName['Message'])
                return siteName
            sites.append(siteName['Value'])

        sites = Utils.list_sanitize(Utils.list_flatten(sites))
        _ = [self.__updateService(s, serviceType) for s in sites]

        # Update Resource table
        for node in NodeInCS:
            if serviceType == "Computing":
                resourceType = CS.getCEType(site, node)
            if node not in resourcesInDB and node is not None:
                try:
                    siteInGOCDB = self.__getServiceEndpointInfo(
                        node)[0]['SITENAME']
                except IndexError:  # No INFO in GOCDB: Node does not exist
                    gLogger.warn(
                        "Node %s is not in GOCDB!! Considering that it does not exists!"
                        % node)
                    continue

                assert (type(siteInGOCDB) == str)
                #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
                res = self.rsClient.addOrModifyResource(
                    node, resourceType, serviceType, site, siteInGOCDB)
                if not res['OK']:
                    gLogger.error(res['Message'])
                    return res
                resourcesInDB.add(node)
Example #8
0
    def __init__(self):

        self.log = gLogger.getSubLogger('PilotsLoggingDB')

        result = getDBParameters('WorkloadManagement/PilotsLoggingDB')
        if not result['OK']:
            raise RuntimeError('Cannot get database parameters: %s' %
                               result['Message'])

        dbParameters = result['Value']
        self.dbHost = dbParameters['Host']
        self.dbPort = dbParameters['Port']
        self.dbUser = dbParameters['User']
        self.dbPass = dbParameters['Password']
        self.dbName = dbParameters['DBName']

        # These are the list of tables that will be created.
        # They can be extended in an extension module
        self.tablesList = getattr(
            Utils.voimport(
                'DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB'),
            'TABLESLIST')

        self.__initializeConnection()
        resp = self.__initializeDB()
        if not resp['OK']:
            raise Exception("Couldn't create tables: " + resp['Message'])
Example #9
0
    def __init__(self,
                 VOExtension,
                 rsDBIn=None,
                 commandCallerIn=None,
                 infoGetterIn=None,
                 WMSAdminIn=None):
        """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

        self.configModule = Utils.voimport(
            "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

        if rsDBIn is not None:
            self.rsDB = rsDBIn
        else:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            self.rsDB = ResourceStatusDB()

        from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
        self.rmDB = ResourceManagementDB()

        if commandCallerIn is not None:
            self.cc = commandCallerIn
        else:
            from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
            self.cc = CommandCaller()

        if infoGetterIn is not None:
            self.ig = infoGetterIn
        else:
            from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
            self.ig = InfoGetter(VOExtension)

        if WMSAdminIn is not None:
            self.WMSAdmin = WMSAdminIn
        else:
            from DIRAC.Core.DISET.RPCClient import RPCClient
            self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

        self.threadPool = ThreadPool(2, 5)

        self.lockObj = threading.RLock()

        self.infoForPanel_res = {}
Example #10
0
def initializeResourceStatusHandler(_serviceInfo):
    '''
    Handler initialization, where we set the ResourceStatusDB as global db, and
    we instantiate the synchronizer.
  '''
    global db
    db = ResourceStatusDB()

    # Publisher is on boxes right now
    #
    #  rmDB = ResourceStatusDB()
    #  cc = CommandCaller()
    #  global VOExtension
    #  VOExtension = getExt()
    #  ig = InfoGetter( VOExtension )
    #  WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
    #  global publisher
    #  publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
    #                         infoGetterIn = ig, WMSAdminIn = WMSAdmin )

    syncModule = Utils.voimport(
        'DIRAC.ResourceStatusSystem.Utilities.Synchronizer')
    syncObject = syncModule.Synchronizer()
    gConfig.addListenerToNewVersionEvent(syncObject.sync)

    return S_OK()
Example #11
0
    def __init__(self, parentLogger=None):

        if not parentLogger:
            parentLogger = gLogger
        self.log = parentLogger.getSubLogger(self.__class__.__name__)

        result = getDBParameters("WorkloadManagement/PilotsLoggingDB")
        if not result["OK"]:
            raise RuntimeError("Cannot get database parameters: %s" % result["Message"])

        dbParameters = result["Value"]
        self.dbHost = dbParameters["Host"]
        self.dbPort = dbParameters["Port"]
        self.dbUser = dbParameters["User"]
        self.dbPass = dbParameters["Password"]
        self.dbName = dbParameters["DBName"]

        # These are the list of tables that will be created.
        # They can be extended in an extension module
        self.tablesList = getattr(Utils.voimport("DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB"), "TABLESLIST")

        self.__initializeConnection()
        resp = self.__initializeDB()
        if not resp["OK"]:
            raise Exception("Couldn't create tables: " + resp["Message"])
Example #12
0
def getValidStatusTypes():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Resources
  '''
  
  DEFAULTS = { 
               'Site'          : { 'StatusType' : "''" },
               'Service'       : { 'StatusType' : "''" },
               'Resource'      : { 'StatusType' : "''" },
               'StorageElement': { 'StatusType' : [ 'Read', 'Write', 'Remove', 'Check' ] }
              }
  
  opHelper = Operations()
  
  sections = opHelper.getSections( 'RSSConfiguration/GeneralConfig/Resources' )
  if not sections[ 'OK' ]:
    return DEFAULTS
  
  result = {}
  for section in sections[ 'Value' ]:
    res = opHelper.getValue( 'RSSConfiguration/GeneralConfig/Resources/%s/StatusType' % section )
    if res is None:
      if DEFAULTS.has_key( section ):
        result[ section ] = { 'StatusType' : DEFAULTS[ section ] }
      else:
        result[ section ] = { 'StatusType' : None }  
    else:
      result[ section ] = { 'StatusType' : Utils.getTypedList( res ) }
      
  return result     
Example #13
0
  def __getPolTypes(self, granularity, status=None, formerStatus=None, newStatus=None,
                    siteType=None, serviceType=None, resourceType=None):
    """Get Policy Types from config that match the given keyword
    arguments"""

    # This dict is constructed to be used with function dictMatch that
    # helps selecting policies. **kwargs are not used due to the fact
    # that it's too dangerous here.
    argsdict = {'Granularity': granularity,
                'Status': status,
                'FormerStatus': formerStatus,
                'NewStatus': newStatus,
                'SiteType': siteType,
                'ServiceType': serviceType,
                'ResourceType': resourceType}

    pTconfig = getTypedDictRootedAt("PolicyTypes")

    pTypes = []

    for pt in pTconfig:
      if Utils.dictMatch(argsdict, pTconfig[pt]):
        pTypes.append(pt)

    for pt_name in pTypes:
      if 'Alarm_PolType' in pt_name:
        pTypes.remove(pt_name)
        pTypes.append('Alarm_PolType')

    return pTypes
Example #14
0
def initializeResourceStatusHandler( _serviceInfo ):
  '''
    Handler initialization, where we set the ResourceStatusDB as global db, and
    we instantiate the synchronizer.
  '''
  global db
  db = ResourceStatusDB()

# Publisher is on boxes right now
#
#  rmDB = ResourceStatusDB()
#  cc = CommandCaller()
#  global VOExtension
#  VOExtension = getExt()
#  ig = InfoGetter( VOExtension )
#  WMSAdmin = RPCClient( "WorkloadStatus/WMSAdministrator" )
#  global publisher
#  publisher = Publisher( VOExtension, dbIn = db, commandCallerIn = cc,
#                         infoGetterIn = ig, WMSAdminIn = WMSAdmin )

  syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer' )
  syncObject = syncModule.Synchronizer()
  gConfig.addListenerToNewVersionEvent( syncObject.sync )
  
  return S_OK()
Example #15
0
def getSpaceTokenEndpoints():
    ''' Get Space Token Endpoints '''

    return Utils.getCSTree('Shares/Disk')


################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
Example #16
0
  def __getPanelsInfo( self, granularity, statusType = None, status = None,
                       formerStatus = None, siteType = None, serviceType = None,
                       resourceType = None, panel_name = None, useNewRes = False ):

    info = []

    # First, select only policies we want.
    argsdict = {'Granularity'  : granularity,
                'StatusType'   : statusType,
                'Status'       : status,
                'FormerStatus' : formerStatus,
                'SiteType'     : siteType,
                'ServiceType'  : serviceType,
                'ResourceType' : resourceType}


    all_policies = getTypedDictRootedAtOperations("Policies")
    selected_policies = []
    for p in all_policies:
      if Utils.dictMatch(argsdict, all_policies[p]):
        selected_policies.append(p)

    for p in selected_policies:                   # For selected policies
      if panel_name in self.C_Policies[p].keys(): # For selected panel_name (arguments)

        toAppend = copy.deepcopy(self.C_Policies[p][panel_name]) # type(toAppend) = list

        # Put CommandIn and args to correct values according to useNewRes
        if useNewRes:
          for panel in toAppend:
            for info_type in panel.keys():

              if type(panel[info_type]) == dict:
                try:
                  panel[info_type]['CommandIn'] = panel[info_type]['CommandInNewRes']
                  del panel[info_type]['CommandInNewRes']
                except KeyError:
                  pass
                try:
                  panel[info_type]['args'] = panel[info_type]['argsNewRes']
                  del panel[info_type]['argsNewRes']
                except KeyError:
                  pass
        else:
          for panel in toAppend:
            for info_type in panel.keys():
              try:
                del panel[info_type]['CommandInNewRes']
              except KeyError:
                pass
              try:
                del panel[info_type]['argsNewRes']
              except KeyError:
                pass

        info.append({p:toAppend})

    return info
Example #17
0
  def __init__( self ):
    """c'tor

    :param self: self reference
    """

    self.log = gLogger.getSubLogger( 'ResourceStatusDB' )

    #These are the list of tables that will be created.
    #They can be extended in an extension module
    self.tablesList = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceStatusDB' ),
                              'TABLESLIST')
    self.tablesListWithID = getattr(Utils.voimport( 'DIRAC.ResourceStatusSystem.DB.ResourceStatusDB' ),
                                    'TABLESLISTWITHID')

    self.extensions = gConfig.getValue( 'DIRAC/Extensions', [] )
    self.__initializeConnection( 'ResourceStatus/ResourceStatusDB' )
    self.__initializeDB()
Example #18
0
    def enforce(self, decissionParams):
        '''
      Enforce policies for given set of keyworkds. To be better explained.
    '''

        ## policy decision point setup #############################################

        self.pdp.setup(decissionParams)

        ## policy decision #########################################################

        resDecisions = self.pdp.takeDecision()
        if not resDecisions['OK']:
            gLogger.error(
                'PEP: Something went wrong, not enforcing policies for %s' %
                decissionParams)
            return resDecisions
        resDecisions = resDecisions['Value']

        # We take from PDP the decision parameters used to find the policies
        decissionParams = resDecisions['decissionParams']
        policyCombinedResult = resDecisions['policyCombinedResult']
        singlePolicyResults = resDecisions['singlePolicyResults']

        for policyActionName, policyActionType in policyCombinedResult[
                'PolicyAction']:

            try:
                actionMod = Utils.voimport(
                    'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' %
                    policyActionType)
            except ImportError:
                gLogger.error('Error importing %s action' % policyActionType)
                continue

            try:
                action = getattr(actionMod, policyActionType)
            except AttributeError:
                gLogger.error('Error importing %s action class' %
                              policyActionType)
                continue

            actionObj = action(policyActionName, decissionParams,
                               policyCombinedResult, singlePolicyResults,
                               self.clients)

            gLogger.debug((policyActionName, policyActionType))

            actionResult = actionObj.run()
            if not actionResult['OK']:
                gLogger.error(actionResult['Message'])

        return S_OK(resDecisions)


################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
Example #19
0
  def policyInvocation( self, granularity = None, name = None,
                        status = None, policy = None, args = None, pName = None,
                        pModule = None, extraArgs = None, commandIn = None ):
    '''
    Invokes a policy:

    1. If :attr:`policy` is None, import the policy module specified
    with :attr:`pModule` (e.g. 'DT_Policy').

      1.1. Create a policy object.

    2. Set the policy arguments (usually :attr:`granularity`,
    :attr:`name`) + :attr:`extraArgs`.

    3. If commandIn is specified (normally it is), use
    :meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
    to get a command object
    '''

    if not policy:

      try:
      
        policyModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.%s' % pModule )
        
      except ImportError:
        _msg = 'Unable to import a policy module named %s, falling back on AlwaysFalse_Policy.' % pModule
        gLogger.warn( _msg )
        policyModule = __import__( 'DIRAC.ResourceStatusSystem.Policy.AlwaysFalse_Policy',
                                   globals(), locals(), ['*'] )
        pModule = 'AlwaysFalse_Policy'
        
      try:
        
        policy = getattr( policyModule, pModule )()
        
      except AttributeError as exc:
        print policyModule, pModule
        raise exc

    if not args:
      args = ( granularity, name )

    if extraArgs:
      args = args + tuple( extraArgs )

    if commandIn:
      commandIn = self.cCaller.setCommandObject( commandIn )
      for clientName, clientInstance in self.clients.items():
        
        self.cCaller.setAPI( commandIn, clientName, clientInstance )

    res = self._innerEval( policy, args, commandIn = commandIn )
    # Just adding the PolicyName to the result of the evaluation of the policy
    res[ 'PolicyName' ] = pName
    return res
Example #20
0
  def __init__(self, VOExtension):
    """
    Standard constructor

    :params:
      :attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
    """

    configModule    = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
    self.C_Policies = copy.deepcopy(configModule.Policies)
Example #21
0
 def _getUsersToNotify(self):
     groups = CS.getTypedDictRootedAtOperations("AssigneeGroups/" +
                                                CS.getSetup()).values()
     concerned_groups = [
         g for g in groups if Utils.dictMatch(self.kw["Params"], g)
     ]
     return [{
         'Users': g['Users'],
         'Notifications': g['Notifications']
     } for g in concerned_groups]
Example #22
0
  def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
               WMSAdminIn = None):
    """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

    self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

    if rsDBIn is not None:
      self.rsDB = rsDBIn
    else:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      self.rsDB = ResourceStatusDB()

    from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
    self.rmDB = ResourceManagementDB()

    if commandCallerIn is not None:
      self.cc = commandCallerIn
    else:
      from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
      self.cc = CommandCaller()

    if infoGetterIn is not None:
      self.ig = infoGetterIn
    else:
      from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
      self.ig = InfoGetter(VOExtension)

    if WMSAdminIn is not None:
      self.WMSAdmin = WMSAdminIn
    else:
      from DIRAC.Core.DISET.RPCClient import RPCClient
      self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

    self.threadPool = ThreadPool( 2, 5 )

    self.lockObj = threading.RLock()

    self.infoForPanel_res = {}
Example #23
0
    def policyInvocation(self, decisionParams, policyDict):
        '''
    Invokes a policy:

    1. If :attr:`policy` is None, import the policy module specified
    with :attr:`pModule` (e.g. 'DT_Policy').

      1.1. Create a policy object.

    2. Set the policy arguments (usually :attr:`granularity`,
    :attr:`name`) + :attr:`extraArgs`.

    3. If commandIn is specified (normally it is), use
    :meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
    to get a command object
    '''

        if 'module' not in policyDict:
            return S_ERROR('Malformed policyDict %s' % policyDict)
        pModuleName = policyDict['module']

        if 'command' not in policyDict:
            return S_ERROR('Malformed policyDict %s' % policyDict)
        pCommand = policyDict['command']

        if 'args' not in policyDict:
            return S_ERROR('Malformed policyDict %s' % policyDict)
        pArgs = policyDict['args']

        try:
            policyModule = Utils.voimport(
                'DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName)
        except ImportError:
            return S_ERROR(
                'Unable to import DIRAC.ResourceStatusSystem.Policy.%s' %
                pModuleName)

        if not hasattr(policyModule, pModuleName):
            return S_ERROR('%s has no attibute %s' %
                           (policyModule, pModuleName))

        policy = getattr(policyModule, pModuleName)()

        command = self.cCaller.commandInvocation(pCommand, pArgs,
                                                 decisionParams, self.clients)
        if not command['OK']:
            return command
        command = command['Value']

        evaluationResult = self.policyEvaluation(policy, command)

        if evaluationResult['OK']:
            evaluationResult['Value']['Policy'] = policyDict

        return evaluationResult
Example #24
0
def getValidPolicyTypes():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/PolicyTypes
  '''
  
  DEFAULTS = [ 'Resource_PolType', 'Alarm_PolType', 'Collective_PolType', 'RealBan_PolType' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/PolicyTypes' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #25
0
    def __init__(self, VOExtension):
        """
    Standard constructor

    :params:
      :attr:`VOExtension`: string - VO extension (e.g. 'LHCb')
    """

        configModule = Utils.voimport(
            "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
        self.C_Policies = copy.deepcopy(configModule.Policies)
Example #26
0
 def __init__( self ):
   """ Constructor. Imports the policy configurations containing the command
   information, among other things. 
   
   examples:
     >>> iGetter = InfoGetter()
   
   """
       
   configModule  = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.Configurations' )
   self.policies = copy.deepcopy( configModule.POLICIESMETA )  
Example #27
0
def getValidServiceTypes():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/ServiceType
  '''
  
  DEFAULTS = [ 'Computing', 'Storage', 'VO-BOX', 'VOMS', 'CondDB' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/ServiceType' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #28
0
def getValidPolicyResult():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/PolicyResult
  '''
  
  DEFAULTS = [ 'Error', 'Unknown', 'Banned', 'Probing', 'Bad', 'Active' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/PolicyResult' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #29
0
def getValidElements():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Granularity
  '''
  
  DEFAULTS = [ 'Site', 'Service', 'Resource', 'StorageElement' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/Granularity' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #30
0
def getValidStatus():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Status
  '''
  
  DEFAULTS = [ 'Active', 'Bad', 'Probing', 'Banned' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/Status' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #31
0
def getValidResourceTypes():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/ResourceType
  '''
  
  DEFAULTS = [ 'CE', 'CREAMCE', 'SE', 'LFC_C', 'LFC_L', 'FTS', 'VOMS' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/ResourceType' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #32
0
def getValidSiteTypes():
  '''
  Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/SiteType
  '''
  
  DEFAULTS = [ 'T1', 'T2', 'T3', 'T4' ]
  
  result = Operations().getValue( 'RSSConfiguration/GeneralConfig/SiteType' )
  if result is not None:
    return Utils.getTypedList( result )
  return DEFAULTS
Example #33
0
  def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"):

    nodesToUpdate = NodeInCS - resourcesInDB
    if len(nodesToUpdate) > 0:
      gLogger.debug(str(NodeInCS))
      gLogger.debug(str(nodesToUpdate))

    # Update Service table
    siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate]
    siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
    #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
    
    sites = []
    for sInGOCDB in siteInGOCDB:
      siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] )
      if not siteName[ 'OK' ]:
        gLogger.error( siteName[ 'Message' ] )
        return siteName
      sites.append( siteName[ 'Value' ] )
    
    sites = Utils.list_sanitize( Utils.list_flatten( sites ) )
    _     = [ self.__updateService(s, serviceType) for s in sites ]

    # Update Resource table
    for node in NodeInCS:
      if serviceType == "Computing":
        resourceType = CS.getCEType(site, node)
      if node not in resourcesInDB and node is not None:
        try:
          siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME']
        except IndexError: # No INFO in GOCDB: Node does not exist
          gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node)
          continue

        assert(type(siteInGOCDB) == str)
        #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
        res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res
        resourcesInDB.add( node )
Example #34
0
  def __init__( self, VOExtension, granularity = None, name = None, status = None, formerStatus = None,
                reason = None, siteType = None, serviceType = None, resourceType = None,
                tokenOwner = None, useNewRes = False ):

    self.VOExtension = VOExtension

    try:
      self.__granularity = Utils.assignOrRaise( granularity, ValidRes, InvalidRes, self, self.__init__ )
    except NameError:
      pass

    self.__name         = name
    self.__status       = Utils.assignOrRaise( status, ValidStatus, InvalidStatus, self, self.__init__ )
    self.__formerStatus = Utils.assignOrRaise( formerStatus, ValidStatus, InvalidStatus, self, self.__init__ )
    self.__reason       = reason
    self.__siteType     = Utils.assignOrRaise( siteType, ValidSiteType, InvalidSiteType, self, self.__init__ )
    self.__serviceType  = Utils.assignOrRaise( serviceType, ValidServiceType, InvalidServiceType, self, self.__init__ )
    self.__resourceType = Utils.assignOrRaise( resourceType, ValidResourceType, InvalidResourceType, self, self.__init__ )

    self.__realBan = False
    if tokenOwner is not None:
      if tokenOwner == 'RS_SVC':
        self.__realBan = True

    self.useNewRes = useNewRes
Example #35
0
File: CS.py Project: bmb/DIRAC
  def getTypedDictRootedAt( path ):
    retval = {}

    opts = gConfig.getOptionsDict( path )
    secs = gConfig.getSections( path )

    if not opts[ 'OK' ]:
      raise CSError, opts[ 'Message' ]
    if not secs[ 'OK' ]:
      raise CSError, secs[ 'Message' ]

    opts = opts[ 'Value' ]
    secs = secs[ 'Value' ]

    for k in opts:
      if opts[ k ].find( "," ) > -1:
        retval[ k ] = [ Utils.typedobj_of_string(e) for e in List.fromChar(opts[k]) ]
      else:
        retval[ k ] = Utils.typedobj_of_string( opts[ k ] )
    for i in secs:
      retval[ i ] = getTypedDictRootedAt( path + "/" + i )
    return retval
Example #36
0
    def __init__(self):
        """c'tor

        :param self: self reference
        """

        super(ResourceStatusDB, self).__init__()

        # These are the list of tables that will be created.
        # They can be extended in an extension module
        self.tablesList = getattr(
            Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"),
            "TABLESLIST")
        self.tablesListWithID = getattr(
            Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"),
            "TABLESLISTWITHID")

        self.extensions = gConfig.getValue("DIRAC/Extensions", [])
        self._initializeConnection("ResourceStatus/ResourceStatusDB")

        # Create required tables
        self._createTablesIfNotThere(self.tablesList)
        self._createTablesIfNotThere(self.tablesListWithID)
Example #37
0
  def policyInvocation( self, decisionParams, policyDict ):
    '''
    Invokes a policy:

    1. If :attr:`policy` is None, import the policy module specified
    with :attr:`pModule` (e.g. 'DT_Policy').

      1.1. Create a policy object.

    2. Set the policy arguments (usually :attr:`granularity`,
    :attr:`name`) + :attr:`extraArgs`.

    3. If commandIn is specified (normally it is), use
    :meth:`DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller.setCommandObject`
    to get a command object
    '''

    if not 'module' in policyDict:
      return S_ERROR( 'Malformed policyDict %s' % policyDict )
    pModuleName = policyDict[ 'module' ]

    if not 'command' in policyDict:
      return S_ERROR( 'Malformed policyDict %s' % policyDict )
    pCommand = policyDict[ 'command' ]
    
    if not 'args' in policyDict:
      return S_ERROR( 'Malformed policyDict %s' % policyDict )
    pArgs = policyDict[ 'args' ]

    try:     
      policyModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName )
    except ImportError:
      return S_ERROR( 'Unable to import DIRAC.ResourceStatusSystem.Policy.%s' % pModuleName )
    
    if not hasattr( policyModule, pModuleName ):
      return S_ERROR( '%s has no attibute %s' % ( policyModule, pModuleName ) ) 
    
    policy  = getattr( policyModule, pModuleName )() 
    
    command = self.cCaller.commandInvocation( pCommand, pArgs, decisionParams, self.clients )
    if not command[ 'OK' ]:
      return command
    command = command[ 'Value' ]
    
    evaluationResult = self.policyEvaluation( policy, command )
    
    if evaluationResult[ 'OK' ]:
      evaluationResult[ 'Value' ][ 'Policy' ] = policyDict
    
    return evaluationResult
Example #38
0
def initializeResourceStatusHandler( _serviceInfo ):
  '''
    Handler initialization, where we set the ResourceStatusDB as global db, and
    we instantiate the synchronizer.
  '''
  
  global db
  db = ResourceStatusDB()

  syncModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Utilities.Synchronizer' )
  syncObject = syncModule.Synchronizer()
  gConfig.addListenerToNewVersionEvent( syncObject.sync )
  
  return S_OK()
Example #39
0
  def __getPolToEval(self, granularity, status=None, formerStatus=None, siteType=None,
                     serviceType=None, resourceType=None, useNewRes=False):

    # This dict is constructed to be used with function dictMatch that
    # helps selecting policies. **kwargs are not used due to the fact
    # that it's too dangerous here.
    argsdict = {'Granularity': granularity,
                'Status': status,
                'FormerStatus': formerStatus,
                'SiteType': siteType,
                'ServiceType': serviceType,
                'ResourceType': resourceType}

    pConfig = getTypedDictRootedAt("Policies")
    pol_to_eval = []

    for p in pConfig:
      if Utils.dictMatch(argsdict, pConfig[p]):
        pol_to_eval.append(p)

    polToEval_Args = []

    for p in pol_to_eval:
      try:
        moduleName = self.C_Policies[p]['module']
      except KeyError:
        moduleName = None
      try:
        ConfirmationPolicy = self.C_Policies[p]['ConfirmationPolicy']
      except KeyError:
        ConfirmationPolicy = None

      if useNewRes:
        try:
          commandIn = self.C_Policies[p]['commandInNewRes']
        except KeyError:
          commandIn = self.C_Policies[p]['commandIn']
        try:
          args = self.C_Policies[p]['argsNewRes']
        except KeyError:
          args = self.C_Policies[p]['args']
      else:
        commandIn = self.C_Policies[p]['commandIn']
        args = self.C_Policies[p]['args']

      polToEval_Args.append({'Name' : p, 'Module' : moduleName, 'args' : args,
                             'ConfirmationPolicy' : ConfirmationPolicy,
                             'commandIn' : commandIn})

    return polToEval_Args
Example #40
0
  def __getPolToEval( self, granularity, statusType = None, status=None,
                      formerStatus=None, siteType=None, serviceType=None,
                      resourceType=None, useNewRes=False ):
    """Returns a possibly empty list of dicts, each dict containing
    enough information to evaluate a given policy"""

    # This dict is constructed to be used with function dictMatch that
    # helps selecting policies. **kwargs are not used due to the fact
    # that it's too dangerous here.
    argsdict = { 'Granularity'  : granularity,
                 'StatusType'   : statusType,
                 'Status'       : status,
                 'FormerStatus' : formerStatus,
                 'SiteType'     : siteType,
                 'ServiceType'  : serviceType,
                 'ResourceType' : resourceType }

    pConfig = getTypedDictRootedAtOperations("Policies")
    pol_to_eval = (p for p in pConfig if Utils.dictMatch(argsdict, pConfig[p]))
    polToEval_Args = []

    for p in pol_to_eval:
      try:
        moduleName = self.C_Policies[p]['module']
      except KeyError:
        moduleName = None
      try:
        ConfirmationPolicy = self.C_Policies[p]['ConfirmationPolicy']
      except KeyError:
        ConfirmationPolicy = None

      if useNewRes:
        try:
          commandIn = self.C_Policies[p]['commandInNewRes']
        except KeyError:
          commandIn = self.C_Policies[p]['commandIn']
        try:
          args = self.C_Policies[p]['argsNewRes']
        except KeyError:
          args = self.C_Policies[p]['args']
      else:
        commandIn = self.C_Policies[p]['commandIn']
        args = self.C_Policies[p]['args']

      polToEval_Args.append({'Name' : p, 'Module' : moduleName, 'args' : args,
                             'ConfirmationPolicy' : ConfirmationPolicy,
                             'commandIn' : commandIn})

    return polToEval_Args
Example #41
0
    def getTypedDictRootedAt(path):
        retval = {}

        opts = gConfig.getOptionsDict(path)
        secs = gConfig.getSections(path)

        if not opts['OK']:
            raise CSError, opts['Message']
        if not secs['OK']:
            raise CSError, secs['Message']

        opts = opts['Value']
        secs = secs['Value']

        for k in opts:
            if opts[k].find(",") > -1:
                retval[k] = [
                    Utils.typedobj_of_string(e) for e in List.fromChar(opts[k])
                ]
            else:
                retval[k] = Utils.typedobj_of_string(opts[k])
        for i in secs:
            retval[i] = getTypedDictRootedAt(path + "/" + i)
        return retval
Example #42
0
    def __init__(self):
        """c'tor

        :param self: self reference
        """

        super(ResourceManagementDB, self).__init__()

        # This is the list of tables that will be created.
        # It can be extended in an extension module
        self.tablesList = getattr(Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceManagementDB"), "TABLESLIST")
        self._initializeConnection("ResourceStatus/ResourceManagementDB")

        # Create required tables
        self._createTablesIfNotThere(self.tablesList)
Example #43
0
  def __loadTestObj(self):
    _module_pre = 'DIRAC.ResourceStatusSystem.SAM.SAMTest.'

    for testType, testDict in self.tests.items():
      moduleName = testDict[ 'module' ]
      args = testDict.get( 'args', {} )
      args.update( testDict[ 'match' ] )
      args[ 'TestType' ] = testType
      try:
        testModule = Utils.voimport( _module_pre + moduleName )
      except ImportError, e:
        gLogger.error( "Unable to import %s, %s" % ( _module_pre + moduleName, e ) )
        continue
      testClass = getattr( testModule, moduleName )
      obj = testClass(args, self.apis)
      testDict[ 'object' ] = obj
Example #44
0
    def __loadTestObj(self):
        _module_pre = 'DIRAC.ResourceStatusSystem.SAM.SAMTest.'

        for testType, testDict in self.tests.items():
            moduleName = testDict['module']
            args = testDict.get('args', {})
            args.update(testDict['match'])
            args['TestType'] = testType
            try:
                testModule = Utils.voimport(_module_pre + moduleName)
            except ImportError, e:
                gLogger.error("Unable to import %s, %s" %
                              (_module_pre + moduleName, e))
                continue
            testClass = getattr(testModule, moduleName)
            obj = testClass(args, self.apis)
            testDict['object'] = obj
Example #45
0
def commandInvocation(commandTuple,
                      pArgs=None,
                      decissionParams=None,
                      clients=None):
    '''
  Returns a command object, given commandTuple

  :params:
    `commandTuple`: a tuple, where commandTuple[0] is a module name and 
    commandTuple[1] is a class name (inside the module)
  '''

    if commandTuple is None:
        return S_OK(None)

    # decission params can be a dictionary passed with all the element parameters
    # used mostly by the PDP to inject all relevant information
    if decissionParams is None:
        decissionParams = {}

    # arguments hardcoded on Configurations.py for the policy
    if pArgs is None:
        pArgs = {}

    try:
        cModule = commandTuple[0]
        cClass = commandTuple[1]
        commandModule = Utils.voimport('DIRAC.ResourceStatusSystem.Command.' +
                                       cModule)
    except ImportError:
        return S_ERROR("Import error for command %s." % (cModule))

    if not hasattr(commandModule, cClass):
        return S_ERROR('%s has no %s' % (cModule, cClass))

    # We merge decision parameters and policy arguments.
    newArgs = copy.deepcopy(decissionParams)
    newArgs.update(pArgs)

    commandObject = getattr(commandModule, cClass)(newArgs, clients)

    return S_OK(commandObject)


################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
Example #46
0
  def setCommandObject( self, comm ):
    """
    Returns a command object, given comm

    :params:
      `comm`: a tuple, where comm[0] is a module name and comm[1] is a class name (inside the module)
    """
    try:
      cModule = comm[0]
      cClass = comm[1]
      commandModule = Utils.voimport("DIRAC.ResourceStatusSystem.Command." + cModule)
    except ImportError:
      gLogger.warn("Command %s/%s not found, using dummy command DoNothing_Command." % (cModule, cClass))
      cClass = "DoNothing_Command"
      commandModule = __import__("DIRAC.ResourceStatusSystem.Command.DoNothing_Command", globals(), locals(), ['*'])

    c = getattr(commandModule, cClass)()

    return c
Example #47
0
def getUsersToNotify(setup, kwargs):
    """Get a list of users to notify (helper function for AlarmPolTypeActions)
  Optional keyword arguments:
  - Granularity
  - SiteType
  - ServiceType
  - ResourceType
  """

    notifications = []
    groups = CS.getTypedDictRootedAt("AssigneeGroups/" + setup)

    for k in groups:
        if Utils.dictMatch(kwargs, groups[k]):
            notifications.append({
                'Users': groups[k]['Users'],
                'Notifications': groups[k]['Notifications']
            })

    return notifications
Example #48
0
 def __getServiceEndpointInfo(self, node):
   #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) )
   res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node )
   if res['OK']:
     res = res[ 'Value' ]
   else:
     gLogger.warn( 'Error getting hostname info for %s' % node )
     return []
       
   if res == []:
     #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) )
     url = Utils.canonicalURL(node)
     res = self.GOCDBClient.getServiceEndpointInfo('hostname', url )
     if res['OK']:
       res = res[ 'Value' ]
     else:
       gLogger.warn( 'Error getting canonical hostname info for %s' % node )
       res = []
     
   return res
Example #49
0
  def __getPolTypes( self, granularity, statusType=None, status=None,
                     formerStatus=None, newStatus=None, siteType=None,
                     serviceType=None, resourceType=None ):
    """Get Policy Types from config that match the given keyword
    arguments. Always returns a generator object, possibly empty."""

    # This dict is constructed to be used with function dictMatch that
    # helps selecting policies. **kwargs are not used due to the fact
    # that it's too dangerous here.
    argsdict = {'Granularity'  : granularity,
                'StatusType'   : statusType,
                'Status'       : status,
                'FormerStatus' : formerStatus,
                'NewStatus'    : newStatus,
                'SiteType'     : siteType,
                'ServiceType'  : serviceType,
                'ResourceType' : resourceType }

    pTconfig = RssConfiguration.getValidPolicyTypes()
    return (pt for pt in pTconfig if Utils.dictMatch(argsdict, pTconfig[pt]))