コード例 #1
0
    def setUp(self):
        # Create a mock of DB class
        self.mock_DB = Mock()

        # Setting mock return value
        self.mock_DB._query.return_value = {'OK': True, 'Value': ''}
        self.mock_DB._update.return_value = {'OK': True, 'Value': ''}

        sys.modules["DIRAC"] = DIRAC.ResourceStatusSystem.test.fake_Logger
        sys.modules[
            "DIRAC.ResourceStatusSystem.Utilities.CS"] = DIRAC.ResourceStatusSystem.test.fake_Logger
        sys.modules[
            "DIRAC.Core.Utilities.SiteCEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
        sys.modules[
            "DIRAC.Core.Utilities.SiteSEMapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger
        sys.modules[
            "DIRAC.Core.Utilities.SitesDIRACGOCDBmapping"] = DIRAC.ResourceStatusSystem.test.fake_Logger

        from LHCbDIRAC.ResourceStatusSystem.Policy import Configurations
        from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB

        # setting mock interface
        self.rmDB = ResourceManagementDB(DBin=self.mock_DB)

        self.mock_DB_1 = Mock()
        self.mock_DB_1._query.return_value = {
            'OK': True,
            'Value': (('VOMS', ), )
        }

        self.rmDB_1 = ResourceManagementDB(DBin=self.mock_DB_1)
コード例 #2
0
ファイル: Publisher.py プロジェクト: zhangxiaomei/DIRAC
    def __init__(self,
                 VOExtension,
                 rsDBIn=None,
                 commandCallerIn=None,
                 infoGetterIn=None,
                 WMSAdminIn=None):
        """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

        self.configModule = Utils.voimport(
            "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

        if rsDBIn is not None:
            self.rsDB = rsDBIn
        else:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            self.rsDB = ResourceStatusDB()

        from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
        self.rmDB = ResourceManagementDB()

        if commandCallerIn is not None:
            self.cc = commandCallerIn
        else:
            from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
            self.cc = CommandCaller()

        if infoGetterIn is not None:
            self.ig = infoGetterIn
        else:
            from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
            self.ig = InfoGetter(VOExtension)

        if WMSAdminIn is not None:
            self.WMSAdmin = WMSAdminIn
        else:
            from DIRAC.Core.DISET.RPCClient import RPCClient
            self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

        self.threadPool = ThreadPool(2, 5)

        self.lockObj = threading.RLock()

        self.infoForPanel_res = {}
コード例 #3
0
ファイル: CS.py プロジェクト: NathalieRauschmayr/DIRAC
def getMailForUser(users):
  from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
  rmDB = ResourceManagementDB()

  if type(users) == str:
    users = [users]
  else:
    raise ValueError

  return S_OK([rmDB.registryGetMailFromLogin(u) for u in users])
コード例 #4
0
ファイル: Synchronizer.py プロジェクト: zhangxiaomei/DIRAC
    def __init__(self, rsDBin=None, rmDBin=None):

        self.rsDB = rsDBin
        self.rmDB = rmDBin

        if self.rsDB == None and self.rmDB == None:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()

        self.GOCDBClient = GOCDBClient()
コード例 #5
0
def initializeResourceManagementHandler(_serviceInfo):
    '''
    Handler initialization, where we set the ResourceManagementDB as global db.
  '''

    global db
    db = ResourceManagementDB()
    # Regenerates DB tables if needed
    db._checkTable()

    syncObject = Synchronizer.Synchronizer()
    gConfig.addListenerToNewVersionEvent(syncObject.sync)

    return S_OK()
コード例 #6
0
def initializeResourceManagementHandler( _serviceInfo ):
  '''
    Handler initialization, where we set the ResourceManagementDB as global db.
  '''
  
  global db
  db = ResourceManagementDB()
  # Regenerates DB tables if needed  
  db._checkTable()

  syncObject = Synchronizer.Synchronizer()
  gConfig.addListenerToNewVersionEvent( syncObject.sync )

  return S_OK()
コード例 #7
0
ファイル: ClientsCacheFeeder.py プロジェクト: vfalbor/DIRAC
  def initialize( self ):
    """ ClientsCacheFeeder initialization
    """

    try:

      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

      self.clientsInvoker = ClientsInvoker()

      VOExtension = getExt()

      configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
                                globals(), locals(), ['*'] )
      commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )

      commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )

      self.commandObjectsList_ClientsCache = []

      self.commandObjectsList_AccountingCache = []

      cc = CommandCaller()

      RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
      RPCAccounting = RPCClient( "Accounting/ReportGenerator" )

      for command in commandsList_ClientsCache:
        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
                            RPCAccounting = RPCAccounting )
        self.commandObjectsList_ClientsCache.append( ( command, cObj ) )

      for command in commandsList_AccountingCache:
        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
        try:
          cArgs = command[2]
        except IndexError:
          cArgs = ()
        self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )

      return S_OK()

    except Exception:
      errorStr = "ClientsCacheFeeder initialization"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )
コード例 #8
0
def initializeResourceManagementHandler(serviceInfo):

    global rsDB
    rsDB = ResourceStatusDB()
    global rmDB
    rmDB = ResourceManagementDB()

    cc = CommandCaller()

    global VOExtension
    VOExtension = getExt()

    ig = InfoGetter(VOExtension)

    WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

    global publisher
    publisher = Publisher(VOExtension,
                          rsDBIn=rsDB,
                          commandCallerIn=cc,
                          infoGetterIn=ig,
                          WMSAdminIn=WMSAdmin)

    #  sync_O = Synchronizer(rsDB)
    #  gConfig.addListenerToNewVersionEvent( sync_O.sync )

    return S_OK()
コード例 #9
0
  def initialize( self ):
    """ Standard constructor
    """

    try:
      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

      self.StorageElementToBeChecked = Queue.Queue()
      self.StorageElementInCheck     = []

      self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
      self.threadPool         = ThreadPool( self.maxNumberOfThreads,
                                            self.maxNumberOfThreads )

      if not self.threadPool:
        self.log.error( 'Can not create Thread Pool' )
        return S_ERROR( 'Can not create Thread Pool' )

      self.setup                = getSetup()[ 'Value' ]
      self.VOExtension          = getExt()
      self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ]
      self.nc                   = NotificationClient()
      self.diracAdmin           = DiracAdmin()
      self.csAPI                = CSAPI()

      for _i in xrange( self.maxNumberOfThreads ):
        self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) )

      return S_OK()

    except Exception:
      errorStr = "StElWriteInspectorAgent initialization"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )
コード例 #10
0
def initializeResourceManagementHandler(_serviceInfo):
    '''
    Handler initialization, where we set the ResourceManagementDB as global db.
  '''

    global db
    db = ResourceManagementDB()

    return S_OK()
コード例 #11
0
ファイル: Publisher.py プロジェクト: NathalieRauschmayr/DIRAC
  def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
               WMSAdminIn = None):
    """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

    self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

    if rsDBIn is not None:
      self.rsDB = rsDBIn
    else:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      self.rsDB = ResourceStatusDB()

    from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
    self.rmDB = ResourceManagementDB()

    if commandCallerIn is not None:
      self.cc = commandCallerIn
    else:
      from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
      self.cc = CommandCaller()

    if infoGetterIn is not None:
      self.ig = infoGetterIn
    else:
      from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
      self.ig = InfoGetter(VOExtension)

    if WMSAdminIn is not None:
      self.WMSAdmin = WMSAdminIn
    else:
      from DIRAC.Core.DISET.RPCClient import RPCClient
      self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

    self.threadPool = ThreadPool( 2, 5 )

    self.lockObj = threading.RLock()

    self.infoForPanel_res = {}
コード例 #12
0
  def __init__( self, rsDBin = None, rmDBin = None ):

    self.rsDB = rsDBin
    self.rmDB = rmDBin

    if self.rsDB == None and self.rmDB == None:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

    self.GOCDBClient = GOCDBClient()
コード例 #13
0
def installDB():
    '''
    Installs Tables.
  '''

    db1 = ResourceStatusDB()
    db2 = ResourceManagementDB()

    res1 = db1._checkTable()
    if not res1['OK']:
        subLogger.error(res1['Message'])
    elif res1['Value']:
        subLogger.info('ResourceStatusDB')
        subLogger.info(res1['Value'])

    res2 = db2._checkTable()
    if not res2['OK']:
        subLogger.error(res2['Message'])
    elif res2['Value']:
        subLogger.info('ResourceManagementDB')
        subLogger.info(res2['Value'])

    return S_OK()
コード例 #14
0
ファイル: RSInspectorAgent.py プロジェクト: vfalbor/DIRAC
    def initialize(self):
        """ Standard constructor
    """

        try:
            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()

            self.ResourcesToBeChecked = Queue.Queue()
            self.ResourceNamesInCheck = []

            self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
            self.threadPool = ThreadPool(self.maxNumberOfThreads,
                                         self.maxNumberOfThreads)

            if not self.threadPool:
                self.log.error('Can not create Thread Pool')
                return S_ERROR('Can not create Thread Pool')

            self.setup = getSetup()['Value']

            self.VOExtension = getExt()

            configModule = __import__(
                self.VOExtension +
                "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(),
                locals(), ['*'])

            self.Resources_check_freq = copy.deepcopy(
                configModule.Resources_check_freq)

            self.nc = NotificationClient()

            self.diracAdmin = DiracAdmin()

            self.csAPI = CSAPI()

            for i in xrange(self.maxNumberOfThreads):
                self.threadPool.generateJobAndQueueIt(self._executeCheck,
                                                      args=(None, ))

            return S_OK()

        except Exception:
            errorStr = "RSInspectorAgent initialization"
            gLogger.exception(errorStr)
            return S_ERROR(errorStr)
コード例 #15
0
ファイル: CleanerAgent.py プロジェクト: vfalbor/DIRAC
    def initialize(self):
        """ CleanerAgent initialization
    """

        try:

            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()
            self.tablesWithHistory = self.rsDB.getTablesWithHistory()
            self.historyTables = [
                x + 'History' for x in self.tablesWithHistory
            ]

            return S_OK()

        except Exception:
            errorStr = "CleanerAgent initialization"
            gLogger.exception(errorStr)
            return S_ERROR(errorStr)
コード例 #16
0
    def enforce(self,
                pdpIn=None,
                rsDBIn=None,
                rmDBIn=None,
                ncIn=None,
                setupIn=None,
                daIn=None,
                csAPIIn=None,
                knownInfo=None):
        """
    enforce policies, using a PDP  (Policy Decision Point), based on

     self.__granularity (optional)

     self.__name (optional)

     self.__status (optional)

     self.__formerStatus (optional)

     self.__reason (optional)

     self.__siteType (optional)

     self.__serviceType (optional)

     self.__realBan (optional)

     self.__user (optional)

     self.__futurePolicyType (optional)

     self.__futureGranularity (optional)

     :params:
       :attr:`pdpIn`: a custom PDP object (optional)

       :attr:`rsDBIn`: a custom (statuses) database object (optional)

       :attr:`rmDBIn`: a custom (management) database object (optional)

       :attr:`setupIn`: a string with the present setup (optional)

       :attr:`ncIn`: a custom notification client object (optional)

       :attr:`daIn`: a custom DiracAdmin object (optional)

       :attr:`csAPIIn`: a custom CSAPI object (optional)

       :attr:`knownInfo`: a string of known provided information (optional)
    """

        #PDP
        if pdpIn is not None:
            pdp = pdpIn
        else:
            # Use standard DIRAC PDP
            from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
            pdp = PDP(self.VOExtension,
                      granularity=self.__granularity,
                      name=self.__name,
                      status=self.__status,
                      formerStatus=self.__formerStatus,
                      reason=self.__reason,
                      siteType=self.__siteType,
                      serviceType=self.__serviceType,
                      resourceType=self.__resourceType,
                      useNewRes=self.useNewRes)

        #DB
        if rsDBIn is not None:
            rsDB = rsDBIn
        else:
            # Use standard DIRAC DB
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            rsDB = ResourceStatusDB()

        if rmDBIn is not None:
            rmDB = rmDBIn
        else:
            # Use standard DIRAC DB
            from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
            rmDB = ResourceManagementDB()

        #setup
        if setupIn is not None:
            setup = setupIn
        else:
            # get present setup
            setup = CS.getSetup()['Value']

        #notification client
        if ncIn is not None:
            nc = ncIn
        else:
            from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
            nc = NotificationClient()

        #DiracAdmin
        if daIn is not None:
            da = daIn
        else:
            from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
            da = DiracAdmin()

        #CSAPI
        if csAPIIn is not None:
            csAPI = csAPIIn
        else:
            from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
            csAPI = CSAPI()

        ###################
        # policy decision #
        ###################

        resDecisions = pdp.takeDecision(knownInfo=knownInfo)
        assert (type(resDecisions) == dict and resDecisions != {})

        res = resDecisions['PolicyCombinedResult']
        actionBaseMod = "DIRAC.ResourceStatusSystem.PolicySystem.Actions"

        # Security mechanism in case there is no PolicyType returned
        if res == {}:
            EmptyPolTypeActions(self.__granularity, self.__name, resDecisions,
                                res)

        else:
            policyType = res['PolicyType']

            if 'Resource_PolType' in policyType:
                m = Utils.voimport(actionBaseMod + ".Resource_PolType",
                                   self.VOExtension)
                m.ResourcePolTypeActions(self.__granularity, self.__name,
                                         resDecisions, res, rsDB, rmDB)

            if 'Alarm_PolType' in policyType:
                m = Utils.voimport(actionBaseMod + ".Alarm_PolType",
                                   self.VOExtension)
                m.AlarmPolTypeActions(self.__name,
                                      res,
                                      nc,
                                      setup,
                                      rsDB,
                                      rmDB,
                                      Granularity=self.__granularity,
                                      SiteType=self.__siteType,
                                      ServiceType=self.__serviceType,
                                      ResourceType=self.__resourceType)

            if 'RealBan_PolType' in policyType and self.__realBan == True:
                m = Utils.voimport(actionBaseMod + ".RealBan_PolType",
                                   self.VOExtension)
                m.RealBanPolTypeActions(self.__granularity, self.__name, res,
                                        da, csAPI, setup)
コード例 #17
0
class ClientsCacheFeederAgent( AgentModule ):

#############################################################################

  def initialize( self ):
    """ ClientsCacheFeederAgent initialization
    """

    try:

      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

      self.clientsInvoker = ClientsInvoker()

      commandsList_ClientsCache = [
        ( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
        ( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
        ( 'ClientsCache_Command', 'DTEverySites_Command' ),
        ( 'ClientsCache_Command', 'DTEveryResources_Command' )
        ]

      commandsList_AccountingCache =  [
        ( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
        ( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
        ( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
        ]

      self.commandObjectsList_ClientsCache = []
      self.commandObjectsList_AccountingCache = []

      cc = CommandCaller()

      RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
      RPCAccounting = RPCClient( "Accounting/ReportGenerator" )

      for command in commandsList_ClientsCache:

        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
                            RPCAccounting = RPCAccounting )
        self.commandObjectsList_ClientsCache.append( ( command, cObj ) )

      for command in commandsList_AccountingCache:
        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
        try:
          cArgs = command[2]
        except IndexError:
          cArgs = ()
        self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )

      return S_OK()

    except Exception:
      errorStr = "ClientsCacheFeederAgent initialization"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )

#############################################################################

  def execute( self ):
    """ The main ClientsCacheFeederAgent execution method
    """

    try:

      now = datetime.datetime.utcnow()

      for co in self.commandObjectsList_ClientsCache:
        try:
          self.clientsInvoker.setCommand( co[1] )
          res = self.clientsInvoker.doCommand()
          for key in res.keys():
            if 'ID' in res[key].keys():
              for value in res[key].keys():
                if value != 'ID':
                  self.rmDB.addOrModifyClientsCacheRes( key.split()[1], co[0][1].split( '_' )[0],
                                                       value, res[key][value], res[key]['ID'] )
            else:
              for value in res[key].keys():
                self.rmDB.addOrModifyClientsCacheRes( key, co[0][1].split( '_' )[0],
                                                     value, res[key][value] )
        except:
          gLogger.exception( "Exception when executing " + co[0][1] )
          continue

      for co in self.commandObjectsList_AccountingCache:
        if co[0][3] == 'Hourly':
          if now.minute >= 10:
            continue
        elif co[0][3] == 'Daily':
          if now.hour >= 1:
            continue

        try:
          co[1].setArgs( co[2] )
          self.clientsInvoker.setCommand( co[1] )
          res = self.clientsInvoker.doCommand()
          plotType = res.keys()[ 0 ]
          for name in res[ plotType ].keys():
            plotName = co[0][1].split( '_' )[0] + '_' + str( co[2][0] )
            self.rmDB.addOrModifyAccountingCacheRes( name, plotType, plotName,
                                                    res[plotType][name] )
        except:
          gLogger.exception( "Exception when executing " + co[ 0 ][ 1 ] )
          continue

      return S_OK()

    except Exception:
      errorStr = "ClientsCacheFeederAgent execution"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )
コード例 #18
0
ファイル: ClientsCacheFeeder.py プロジェクト: vfalbor/DIRAC
class ClientsCacheFeeder( AgentModule ):

#############################################################################

  def initialize( self ):
    """ ClientsCacheFeeder initialization
    """

    try:

      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

      self.clientsInvoker = ClientsInvoker()

      VOExtension = getExt()

      configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
                                globals(), locals(), ['*'] )
      commandsList_ClientsCache = copy.deepcopy( configModule.Commands_ClientsCache )

      commandsList_AccountingCache = copy.deepcopy( configModule.Commands_AccountingCache )

      self.commandObjectsList_ClientsCache = []

      self.commandObjectsList_AccountingCache = []

      cc = CommandCaller()

      RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
      RPCAccounting = RPCClient( "Accounting/ReportGenerator" )

      for command in commandsList_ClientsCache:
        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
                            RPCAccounting = RPCAccounting )
        self.commandObjectsList_ClientsCache.append( ( command, cObj ) )

      for command in commandsList_AccountingCache:
        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
        try:
          cArgs = command[2]
        except IndexError:
          cArgs = ()
        self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )

      return S_OK()

    except Exception:
      errorStr = "ClientsCacheFeeder initialization"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )

#############################################################################

  def execute( self ):
    """ The main ClientsCacheFeeder execution method
    """

    try:

      now = datetime.datetime.utcnow()

      for co in self.commandObjectsList_ClientsCache:
        try:
          self.clientsInvoker.setCommand( co[1] )
          res = self.clientsInvoker.doCommand()
          for key in res.keys():
            if 'ID' in res[key].keys():
              for value in res[key].keys():
                if value != 'ID':
                  self.rmDB.addOrModifyClientsCacheRes( key.split()[1], co[0][1].split( '_' )[0],
                                                       value, res[key][value], res[key]['ID'] )
            else:
              for value in res[key].keys():
                self.rmDB.addOrModifyClientsCacheRes( key, co[0][1].split( '_' )[0],
                                                     value, res[key][value] )
        except:
          gLogger.exception( "Exception when executing " + co[0][1] )
          continue

      for co in self.commandObjectsList_AccountingCache:
        if co[0][3] == 'Hourly':
          if now.minute >= 10:
            continue
        elif co[0][3] == 'Daily':
          if now.hour >= 1:
            continue

        try:
          co[1].setArgs( co[2] )
          self.clientsInvoker.setCommand( co[1] )
          res = self.clientsInvoker.doCommand()
          plotType = res.keys()[0]
          for name in res[plotType].keys():
            plotName = co[0][1].split( '_' )[0] + '_' + str( co[2][0] )
            self.rmDB.addOrModifyAccountingCacheRes( name, plotType, plotName,
                                                    res[plotType][name] )
        except:
          gLogger.exception( "Exception when executing " + co[0][1] )
          continue

      return S_OK()

    except Exception:
      errorStr = "ClientsCacheFeeder execution"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )
コード例 #19
0
ファイル: Publisher.py プロジェクト: NathalieRauschmayr/DIRAC
class Publisher:
  """
  Class Publisher is in charge of getting dispersed information, to be published on the web.
  """

#############################################################################

  def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
               WMSAdminIn = None):
    """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

    self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

    if rsDBIn is not None:
      self.rsDB = rsDBIn
    else:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      self.rsDB = ResourceStatusDB()

    from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
    self.rmDB = ResourceManagementDB()

    if commandCallerIn is not None:
      self.cc = commandCallerIn
    else:
      from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
      self.cc = CommandCaller()

    if infoGetterIn is not None:
      self.ig = infoGetterIn
    else:
      from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
      self.ig = InfoGetter(VOExtension)

    if WMSAdminIn is not None:
      self.WMSAdmin = WMSAdminIn
    else:
      from DIRAC.Core.DISET.RPCClient import RPCClient
      self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

    self.threadPool = ThreadPool( 2, 5 )

    self.lockObj = threading.RLock()

    self.infoForPanel_res = {}

#############################################################################

  def getInfo(self, granularity, name, useNewRes = False):
    """
    Standard method to get all the info to be published

    This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
    with 2-5 threads. The threaded method is
    :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`

    :params:
      :attr:`granularity`: string - a ValidRes

      :attr:`name`: string - name of the Validres

      :attr:`useNewRes`: boolean. When set to true, will get new results,
      otherwise it will get cached results (where available).
    """

    if granularity not in ValidRes:
      raise InvalidRes, Utils.where(self, self.getInfo)

    self.infoForPanel_res = {}

    status = None
    formerStatus = None
    siteType = None
    serviceType = None
    resourceType = None

    if granularity in ('Resource', 'Resources'):
      try:
        resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'],
                                              resourceName = name)[0][0]
      except IndexError:
        return "%s does not exist!" %name

    if granularity in ('StorageElement', 'StorageElements'):
      try:
        siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'],
                                              storageElementName = name)[0][0]
      except IndexError:
        return "%s does not exist!" %name

    paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status',
                  'RSS Status', 'Reason', 'Description']

    infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status,
                                       formerStatus = formerStatus, siteType = siteType,
                                       serviceType = serviceType, resourceType = resourceType,
                                       useNewRes = useNewRes)[0]['Panels']
    infoToGet_res = {}

    recordsList = []

    infosForPolicy = {}

    for panel in infoToGet.keys():

      (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel)

      if not self._resExist(granularityForPanel, nameForPanel):
#        completeInfoForPanel_res = None
        continue

      #take composite RSS result for name
      nameStatus_res = self._getStatus(nameForPanel, panel)

      recordBase = [None, None, None, None, None, None, None, None]

      recordBase[1] = panel.replace('_Panel', '')
      recordBase[2] = nameForPanel #nameForPanel
      try:
        recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status
      except:
        pass
      recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status

      record = copy.deepcopy(recordBase)
      record[0] = 'ResultsForResource'

      recordsList.append(record)

      #take info that goes into the panel
      infoForPanel = infoToGet[panel]

      for info in infoForPanel:

        self.threadPool.generateJobAndQueueIt(self.getInfoForPanel,
                                              args = (info, granularityForPanel, nameForPanel) )

      self.threadPool.processAllResults()

      for policy in [x.keys()[0] for x in infoForPanel]:
        record = copy.deepcopy(recordBase)
        record[0] = 'SpecificInformation'
        record[3] = policy #policyName
        record[4] = None #DIRAC Status
        record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy
        record[6] = self.infoForPanel_res[policy]['Reason'] #Reason
        record[7] = self.infoForPanel_res[policy]['desc'] #Description
        recordsList.append(record)

        infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']

    infoToGet_res['TotalRecords'] = len(recordsList)
    infoToGet_res['ParameterNames'] = paramNames
    infoToGet_res['Records'] = recordsList

    infoToGet_res['Extras'] = infosForPolicy

    return infoToGet_res

#############################################################################

  def getInfoForPanel(self, info, granularityForPanel, nameForPanel):

    #get single RSS policy results
    policyResToGet = info.keys()[0]
    pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet)
    if pol_res != []:
      pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]}
    else:
      pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'}
    self.lockObj.acquire()
    try:
      self.infoForPanel_res[policyResToGet] = pol_res_dict
    finally:
      self.lockObj.release()

    #get policy description
    desc = self._getPolicyDesc(policyResToGet)

    #get other info
    othersInfo = info.values()[0]
    if not isinstance(othersInfo, list):
      othersInfo = [othersInfo]

    info_res = {}

    for oi in othersInfo:
      format_ = oi.keys()[0]
      what = oi.values()[0]

      info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what)

      info_res[format_] = info_bit_got

    self.lockObj.acquire()
    try:
      self.infoForPanel_res[policyResToGet]['infos'] = info_res
      self.infoForPanel_res[policyResToGet]['desc'] = desc
    finally:
      self.lockObj.release()

#############################################################################

  def _getStatus(self, name, panel):

    #get RSS status
    RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]

    #get DIRAC status
    if panel in ('Site_Panel', 'SE_Panel'):

      if panel == 'Site_Panel':
        DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
        if DIRACStatus['OK']:
          DIRACStatus = DIRACStatus['Value'][name].pop()[0]
        else:
          raise RSSException, Utils.where(self, self._getStatus)

      elif panel == 'SE_Panel':
        ra = getStorageElementStatus(name, 'ReadAccess')['Value']
        wa = getStorageElementStatus(name, 'WriteAccess')['Value']
        DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}

      status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } }

    else:
      status = { name : { 'RSSStatus': RSSStatus} }


    return status

#############################################################################

  def _getInfo(self, granularity, name, format_, what):

    if format_ == 'RSS':
      info_bit_got = self._getInfoFromRSSDB(name, what)
    else:
      if isinstance(what, dict):
        command = what['CommandIn']
        extraArgs = what['args']
      else:
        command = what
        extraArgs = None

      info_bit_got = self.cc.commandInvocation(granularity, name, None,
                                               None, command, extraArgs)

      try:
        info_bit_got = info_bit_got['Result']
      except:
        pass

    return info_bit_got

#############################################################################

  def _getInfoFromRSSDB(self, name, what):

    paramsL = ['Status']

    siteName = None
    serviceName = None
    resourceName = None
    storageElementName = None
    serviceType = None
    gridSiteName = None

    if what == 'ServiceOfSite':
      gran = 'Service'
      paramsL.insert(0, 'ServiceName')
      paramsL.append('Reason')
      siteName = name
    elif what == 'ResOfCompService':
      gran = 'Resources'
      paramsL.insert(0, 'ResourceName')
      paramsL.append('Reason')
      serviceType = name.split('@')[0]
      gridSiteName = getGOCSiteName(name.split('@')[1])
      if not gridSiteName['OK']:
        raise RSSException, gridSiteName['Message']
      gridSiteName = gridSiteName['Value']
    elif what == 'ResOfStorService':
      gran = 'Resources'
      paramsL.insert(0, 'ResourceName')
      paramsL.append('Reason')
      serviceType = name.split('@')[0]
      gridSiteName = getGOCSiteName(name.split('@')[1])
      if not gridSiteName['OK']:
        raise RSSException, gridSiteName['Message']
      gridSiteName = gridSiteName['Value']
    elif what == 'ResOfStorEl':
      gran = 'StorageElements'
      paramsL.insert(0, 'ResourceName')
      paramsL.append('Reason')
      storageElementName = name
    elif what == 'StorageElementsOfSite':
      gran = 'StorageElements'
      paramsL.insert(0, 'StorageElementName')
      paramsL.append('Reason')
      if '@' in name:
        DIRACsiteName = name.split('@').pop()
      else:
        DIRACsiteName = name
      gridSiteName = getGOCSiteName(DIRACsiteName)
      if not gridSiteName['OK']:
        raise RSSException, gridSiteName['Message']
      gridSiteName = gridSiteName['Value']
    elif what == 'Site_Panel':
      gran = 'Site'
      paramsL.insert(0, 'SiteName')
      siteName = name
    elif what == 'Service_Computing_Panel':
      gran = 'Service'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Service_Storage_Panel':
      gran = 'Service'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Service_VO-BOX_Panel':
      gran = 'Services'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Service_VOMS_Panel':
      gran = 'Services'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Resource_Panel':
      gran = 'Resource'
      paramsL.insert(0, 'ResourceName')
      resourceName = name
    elif what == 'SE_Panel':
      gran = 'StorageElement'
      paramsL.insert(0, 'StorageElementName')
      storageElementName = name

    info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName,
                                               serviceName = serviceName, serviceType = serviceType,
                                               resourceName = resourceName,
                                               storageElementName = storageElementName,
                                               gridSiteName = gridSiteName)

    return info_bit_got

#############################################################################

  def _getPolicyDesc(self, policyName):

    return self.configModule.Policies[policyName]['Description']

#############################################################################

  def __getNameForPanel(self, granularity, name, panel):

    if granularity in ('Site', 'Sites'):
      if panel == 'Service_Computing_Panel':
        granularity = 'Service'
        name = 'Computing@' + name
      elif panel == 'Service_Storage_Panel':
        granularity = 'Service'
        name = 'Storage@' + name
      elif panel == 'OtherServices_Panel':
        granularity = 'Service'
        name = 'OtherS@' + name
      elif panel == 'Service_VOMS_Panel':
        granularity = 'Service'
        name = 'VOMS@' + name
      elif panel == 'Service_VO-BOX_Panel':
        granularity = 'Service'
        name = 'VO-BOX@' + name
#      else:
#        granularity = granularity
#        name = name
#    else:
#      granularity = granularity
#      name = name

    return (granularity, name)

#############################################################################

  def _resExist(self, granularity, name):

    siteName = None
    serviceName = None
    resourceName = None
    storageElementName = None

    if granularity in ('Site', 'Sites'):
      siteName = name
    elif granularity in ('Service', 'Services'):
      serviceName = name
    elif granularity in ('Resource', 'Resources'):
      resourceName = name
    elif granularity in ('StorageElement', 'StorageElements'):
      storageElementName = name

    res = self.rsDB.getMonitoredsList(granularity, siteName = siteName,
                                      serviceName = serviceName, resourceName = resourceName,
                                      storageElementName = storageElementName)

    if res == []:
      return False
    else:
      return True
コード例 #20
0
  def initialize( self ):
    """ ClientsCacheFeederAgent initialization
    """

    try:

      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

      self.clientsInvoker = ClientsInvoker()

      commandsList_ClientsCache = [
        ( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ),
        ( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
        ( 'ClientsCache_Command', 'DTEverySites_Command' ),
        ( 'ClientsCache_Command', 'DTEveryResources_Command' )
        ]

      commandsList_AccountingCache =  [
        ( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ),
        ( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ),
        ( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ),
        ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ),
        ]

      self.commandObjectsList_ClientsCache = []
      self.commandObjectsList_AccountingCache = []

      cc = CommandCaller()

      RPCWMSAdmin = RPCClient( "WorkloadManagement/WMSAdministrator" )
      RPCAccounting = RPCClient( "Accounting/ReportGenerator" )

      for command in commandsList_ClientsCache:

        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCWMSAdmin = RPCWMSAdmin,
                            RPCAccounting = RPCAccounting )
        self.commandObjectsList_ClientsCache.append( ( command, cObj ) )

      for command in commandsList_AccountingCache:
        cObj = cc.setCommandObject( command )
        cc.setCommandClient( command, cObj, RPCAccounting = RPCAccounting )
        try:
          cArgs = command[2]
        except IndexError:
          cArgs = ()
        self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )

      return S_OK()

    except Exception:
      errorStr = "ClientsCacheFeederAgent initialization"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )
コード例 #21
0
class ClientsCacheFeederAgent(AgentModule):

    #############################################################################

    def initialize(self):
        """ ClientsCacheFeederAgent initialization
    """

        try:

            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()

            self.clientsInvoker = ClientsInvoker()

            commandsList_ClientsCache = [
                ('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
                ('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
                ('ClientsCache_Command', 'DTEverySites_Command'),
                ('ClientsCache_Command', 'DTEveryResources_Command')
            ]

            commandsList_AccountingCache = [
                ('AccountingCache_Command',
                 'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
                ('AccountingCache_Command',
                 'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
                ('AccountingCache_Command',
                 'TransferQualityByDestSplittedSite_Command', (24, ),
                 'Hourly'),
                ('AccountingCache_Command',
                 'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
                 (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
                 (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
            ]

            self.commandObjectsList_ClientsCache = []
            self.commandObjectsList_AccountingCache = []

            cc = CommandCaller()

            RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
            RPCAccounting = RPCClient("Accounting/ReportGenerator")

            for command in commandsList_ClientsCache:

                cObj = cc.setCommandObject(command)
                cc.setCommandClient(command,
                                    cObj,
                                    RPCWMSAdmin=RPCWMSAdmin,
                                    RPCAccounting=RPCAccounting)
                self.commandObjectsList_ClientsCache.append((command, cObj))

            for command in commandsList_AccountingCache:
                cObj = cc.setCommandObject(command)
                cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
                try:
                    cArgs = command[2]
                except IndexError:
                    cArgs = ()
                self.commandObjectsList_AccountingCache.append(
                    (command, cObj, cArgs))

            return S_OK()

        except Exception:
            errorStr = "ClientsCacheFeederAgent initialization"
            gLogger.exception(errorStr)
            return S_ERROR(errorStr)

#############################################################################

    def execute(self):
        """ The main ClientsCacheFeederAgent execution method
    """

        try:

            now = datetime.datetime.utcnow()

            for co in self.commandObjectsList_ClientsCache:
                try:
                    self.clientsInvoker.setCommand(co[1])
                    res = self.clientsInvoker.doCommand()
                    for key in res.keys():
                        if 'ID' in res[key].keys():
                            for value in res[key].keys():
                                if value != 'ID':
                                    self.rmDB.addOrModifyClientsCacheRes(
                                        key.split()[1], co[0][1].split('_')[0],
                                        value, res[key][value], res[key]['ID'])
                        else:
                            for value in res[key].keys():
                                self.rmDB.addOrModifyClientsCacheRes(
                                    key, co[0][1].split('_')[0], value,
                                    res[key][value])
                except:
                    gLogger.exception("Exception when executing " + co[0][1])
                    continue

            for co in self.commandObjectsList_AccountingCache:
                if co[0][3] == 'Hourly':
                    if now.minute >= 10:
                        continue
                elif co[0][3] == 'Daily':
                    if now.hour >= 1:
                        continue

                try:
                    co[1].setArgs(co[2])
                    self.clientsInvoker.setCommand(co[1])
                    res = self.clientsInvoker.doCommand()
                    plotType = res.keys()[0]
                    for name in res[plotType].keys():
                        plotName = co[0][1].split('_')[0] + '_' + str(co[2][0])
                        self.rmDB.addOrModifyAccountingCacheRes(
                            name, plotType, plotName, res[plotType][name])
                except:
                    gLogger.exception("Exception when executing " + co[0][1])
                    continue

            return S_OK()

        except Exception:
            errorStr = "ClientsCacheFeederAgent execution"
            gLogger.exception(errorStr)
            return S_ERROR(errorStr)
コード例 #22
0
    def initialize(self):
        """ ClientsCacheFeederAgent initialization
    """

        try:

            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()

            self.clientsInvoker = ClientsInvoker()

            commandsList_ClientsCache = [
                ('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'),
                ('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'),
                ('ClientsCache_Command', 'DTEverySites_Command'),
                ('ClientsCache_Command', 'DTEveryResources_Command')
            ]

            commandsList_AccountingCache = [
                ('AccountingCache_Command',
                 'TransferQualityByDestSplitted_Command', (2, ), 'Always'),
                ('AccountingCache_Command',
                 'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'),
                ('AccountingCache_Command',
                 'TransferQualityByDestSplittedSite_Command', (24, ),
                 'Hourly'),
                ('AccountingCache_Command',
                 'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command',
                 (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command', 'FailedPilotsByCESplitted_Command',
                 (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'),
                ('AccountingCache_Command',
                 'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'),
            ]

            self.commandObjectsList_ClientsCache = []
            self.commandObjectsList_AccountingCache = []

            cc = CommandCaller()

            RPCWMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
            RPCAccounting = RPCClient("Accounting/ReportGenerator")

            for command in commandsList_ClientsCache:

                cObj = cc.setCommandObject(command)
                cc.setCommandClient(command,
                                    cObj,
                                    RPCWMSAdmin=RPCWMSAdmin,
                                    RPCAccounting=RPCAccounting)
                self.commandObjectsList_ClientsCache.append((command, cObj))

            for command in commandsList_AccountingCache:
                cObj = cc.setCommandObject(command)
                cc.setCommandClient(command, cObj, RPCAccounting=RPCAccounting)
                try:
                    cArgs = command[2]
                except IndexError:
                    cArgs = ()
                self.commandObjectsList_AccountingCache.append(
                    (command, cObj, cArgs))

            return S_OK()

        except Exception:
            errorStr = "ClientsCacheFeederAgent initialization"
            gLogger.exception(errorStr)
            return S_ERROR(errorStr)
コード例 #23
0
class Synchronizer:

#############################################################################

  def __init__( self, rsDBin = None, rmDBin = None ):

    self.rsDB = rsDBin
    self.rmDB = rmDBin

    if self.rsDB == None and self.rmDB == None:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

    self.GOCDBClient = GOCDBClient()

#############################################################################

#  def sync(self, thingsToSync = None, fake_param = None):
  def sync( self, _a, _b ):
    """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """

    thingsToSync = ['Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers']

    gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ' '.join( x for x in thingsToSync ) ) )

    for thing in thingsToSync:
      getattr( self, '_sync' + thing )()

    return S_OK()

#############################################################################

  def _syncUtils( self ):
    """
    Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
    """

    statusIn = self.rsDB.getStatusList()
    #delete status not more in Utils
    for stIn in statusIn:
      if stIn not in ValidStatus:
        self.rsDB.removeStatus( stIn )
    #Add new status
    for s in ValidStatus:
      if s not in statusIn:
        self.rsDB.addStatus( s )

    for g in ( 'Site', 'Service', 'Resource' ):
      typeIn = self.rsDB.getTypesList( g )
      if g == 'Site':
        typesList = ValidSiteType
      elif g == 'Service':
        typesList = ValidServiceType
      if g == 'Resource':
        typesList = ValidResourceType
      #delete types not more in Utils
      for tIn in typeIn:
        if tIn not in typesList:
          self.rsDB.removeType( g, tIn )
      #Add new types
      for t in typesList:
        if t not in typeIn:
          self.rsDB.addType( g, t )

#############################################################################

  def _syncSites( self ):
    """
    Sync DB content with sites that are in the CS
    """

    # sites in the DB now
    sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] )
    sitesIn = [s[0] for s in sitesIn]

    # sites in CS now
    sitesList = getSites()['Value']

    try:
      sitesList.remove( 'LCG.Dummy.ch' )
    except ValueError:
      pass

    # remove sites from the DB not more in the CS
    for site in sitesIn:
      if site not in sitesList:
        self.rsDB.removeSite( site )

    # add to DB what is in CS now and wasn't before
    for site in sitesList:
      if site not in sitesIn:
        # DIRAC Tier
        tier = getSiteTier( site )['Value'][0]
        if tier == 0 or tier == '0':
          t = 'T0'
        elif tier == 1 or tier == '1':
          t = 'T1'
        elif tier == 3 or tier == '3':
          t = 'T3'
        else:
          t = 'T2'

        #Grid Name of the site
        gridSiteName = getGOCSiteName( site )
        if not gridSiteName['OK']:
          raise RSSException, gridSiteName['Message']
        gridSiteName = gridSiteName['Value']

        #Grid Tier (with a workaround!)
        DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
        if not DIRACSitesOfGridSites['OK']:
          raise RSSException, DIRACSitesOfGridSites['Message']
        DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
        if len( DIRACSitesOfGridSites ) == 1:
          gt = t
        else:
          gt = self.__getGOCTier( DIRACSitesOfGridSites )

        self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init',
                                  datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                  datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )

        self.rsDB.addOrModifyGridSite( gridSiteName, gt )

        sitesIn.append( site )

#############################################################################

  def _syncVOBOX( self ):
    """
    Sync DB content with VOBoxes
    """

    # services in the DB now
    servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
    servicesIn = [s[0] for s in servicesIn]

    for site in ['LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es',
                 'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl']:
      service = 'VO-BOX@' + site
      if service not in servicesIn:
        self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init',
                                     datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                     datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )


#############################################################################

  def _syncResources( self ):

    # resources in the DB now
    resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] )
    resourcesIn = [r[0] for r in resourcesIn]

    # services in the DB now
    servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
    servicesIn = [s[0] for s in servicesIn]

    # Site-CE mapping in CS now
    siteCE = getSiteCEMapping( 'LCG' )['Value']
    # Site-SE mapping in CS now
    siteSE = getSiteSEMapping( 'LCG' )['Value']

    # CEs in CS now
    CEList = []
    for i in siteCE.values():
      for ce in i:
        if ce is None:
          continue
        CEList.append( ce )

    # SEs in CS now
    SEList = []
    for i in siteSE.values():
      for x in i:
        SEList.append( x )

    # SE Nodes in CS now
    SENodeList = []
    for SE in SEList:
      node = getSENodes( SE )['Value'][0]
      if node is None:
        continue
      if node not in SENodeList:
        SENodeList.append( node )

    # LFC Nodes in CS now
    LFCNodeList_L = []
    LFCNodeList_C = []
    for site in getLFCSites()['Value']:
      for readable in ( 'ReadOnly', 'ReadWrite' ):
        LFCNode = getLFCNode( site, readable )['Value']
        if LFCNode is None or LFCNode == []:
          continue
        LFCNode = LFCNode[0]
        if readable == 'ReadWrite':
          if LFCNode not in LFCNodeList_C:
            LFCNodeList_C.append( LFCNode )
        elif readable == 'ReadOnly':
          if LFCNode not in LFCNodeList_L:
            LFCNodeList_L.append( LFCNode )

    # FTS Nodes in CS now
    FTSNodeList = []
    sitesWithFTS = getFTSSites()
    for site in sitesWithFTS['Value']:
      fts = getFTSEndpoint( site )['Value']
      if fts is None or fts == []:
        continue
      fts = fts[0]
      if fts not in FTSNodeList:
        FTSNodeList.append( fts )

    # VOMS Nodes in CS now
    VOMSNodeList = getVOMSEndpoints()['Value']


    # complete list of resources in CS now
    resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList

    # list of services in CS now (to be done)
    servicesList = []

    #remove resources no more in the CS
    for res in resourcesIn:
      if res not in resourcesList:
        self.rsDB.removeResource( res )
        self.rsDB.removeStorageElement( resourceName = res )


    # add to DB what is in CS now and wasn't before

    # CEs
    for site in siteCE.keys():
      if site == 'LCG.Dummy.ch':
        continue
      for ce in siteCE[site]:
        if ce is None:
          continue
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce )
        if not siteInGOCDB['OK']:
          raise RSSException, siteInGOCDB['Message']
        if siteInGOCDB['Value'] == []:
          trueName = socket.gethostbyname_ex( ce )[0]
          siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
        try:
          siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
        except IndexError:
          continue
        serviceType = 'Computing'
        service = serviceType + '@' + site

        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )

        if ce not in resourcesIn:
          CEType = getCEType( site, ce )['Value']
          ceType = 'CE'
          if CEType == 'CREAM':
            ceType = 'CREAMCE'
          self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init',
                                        datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                        datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          resourcesIn.append( ce )

    # SRMs
    for srm in SENodeList:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( srm )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )

      if srm not in resourcesIn and srm is not None:
        self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( srm )

    # LFC_C
    for lfc in LFCNodeList_C:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( lfc )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if lfc not in resourcesIn and lfc is not None:
        self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( lfc )

    # LFC_L
    for lfc in LFCNodeList_L:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( lfc )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if lfc not in resourcesIn and lfc is not None:
        self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( lfc )


    # FTSs
    for fts in FTSNodeList:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( fts )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if fts not in resourcesIn and fts is not None:
        self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( fts )

    # VOMSs
    for voms in VOMSNodeList:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( voms )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      site = siteInDIRAC['Value']
      serviceType = 'VOMS'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if voms not in resourcesIn and voms is not None:
        self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( voms )

    #remove services no more in the CS
    for ser in servicesIn:
      if ser not in servicesList:
        serType = ser.split( '@' )[0]
        if serType != 'VO-BOX':
          self.rsDB.removeService( ser )
          self.rsDB.removeResource( serviceName = ser )
          site = ser.split( '@' )[1]
          if serType == 'Storage':
            self.rsDB.removeStorageElement( siteName = site )


#############################################################################

  def _syncStorageElements( self ):

    # Get StorageElements from the CS
    SEs = getStorageElements()
    if not SEs['OK']:
      raise RSSException, SEs['Message']
    SEs = SEs['Value']

    for access in ( 'Read', 'Write' ):

      storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access,
                                                       paramsList = [ 'StorageElementName' ] )
      try:
        storageElementsIn = [ x[ 0 ] for x in storageElementsIn ]
      except IndexError:
        pass

      #remove storageElements no more in the CS
      for se in storageElementsIn:
        if se not in SEs:
          self.rsDB.removeStorageElement( storageElementName = se, resourceName = None, access = access )

      #Add new storage Elements
      for SE in SEs:
        srm = getSENodes( SE )[ 'Value' ][ 0 ]
        if srm == None:
          continue
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
        if not siteInGOCDB[ 'OK' ]:
          raise RSSException, siteInGOCDB[ 'Message' ]
        if siteInGOCDB[ 'Value' ] == []:
          continue
        siteInGOCDB = siteInGOCDB[ 'Value' ][ 0 ][ 'SITENAME' ]

        if SE not in storageElementsIn:
          self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init',
                                               datetime.datetime.utcnow().replace( microsecond = 0 ),
                                              'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ),
                                               access = access )
          storageElementsIn.append( SE )

#############################################################################

  def __getGOCTier( self, sitesList ):

    gridTier = 3

    for site in sitesList:

      tier = getSiteTier( site )['Value'][0]

      if tier == 0 or tier == '0':
        tn = 0
      elif tier == 1 or tier == '1':
        tn = 1
      elif tier == 3 or tier == '3':
        tn = 3
      else:
        tn = 2

      if tn < gridTier:
        gridTier = tn

    if gridTier == 0:
      gt = 'T0'
    elif gridTier == 1:
      gt = 'T1'
    elif gridTier == 3:
      gt = 'T3'
    else:
      gt = 'T2'

    return gt

#############################################################################

  def _syncRegistryUsers(self):
    from DIRAC.ResourceStatusSystem.Utilities import CS
    users = CS.getTypedDictRootedAt("Users", root= "/Registry")
    for u in users:
      if type(users[u]['DN']) == list:
        users[u]['DN'] = users[u]['DN'][0]
      if type(users[u]['Email']) == list:
        users[u]['Email'] = users[u]['Email'][0]

      users[u]['DN'] = users[u]['DN'].split('=')[-1]
      self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())
コード例 #24
0
ファイル: PEP.py プロジェクト: vfalbor/DIRAC
  def enforce(self, pdpIn = None, rsDBIn = None, rmDBIn = None, ncIn = None, setupIn = None,
              daIn = None, csAPIIn = None, knownInfo = None):
    """
    enforce policies, using a PDP  (Policy Decision Point), based on

     self.__granularity (optional)

     self.__name (optional)

     self.__status (optional)

     self.__formerStatus (optional)

     self.__reason (optional)

     self.__siteType (optional)

     self.__serviceType (optional)

     self.__realBan (optional)

     self.__user (optional)

     self.__futurePolicyType (optional)

     self.__futureGranularity (optional)

     :params:
       :attr:`pdpIn`: a custom PDP object (optional)

       :attr:`rsDBIn`: a custom (statuses) database object (optional)

       :attr:`rmDBIn`: a custom (management) database object (optional)

       :attr:`setupIn`: a string with the present setup (optional)

       :attr:`ncIn`: a custom notification client object (optional)

       :attr:`daIn`: a custom DiracAdmin object (optional)

       :attr:`csAPIIn`: a custom CSAPI object (optional)

       :attr:`knownInfo`: a string of known provided information (optional)
    """

    #PDP
    if pdpIn is not None:
      pdp = pdpIn
    else:
      # Use standard DIRAC PDP
      from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP
      pdp = PDP(self.VOExtension, granularity = self.__granularity, name = self.__name,
                status = self.__status, formerStatus = self.__formerStatus, reason = self.__reason,
                siteType = self.__siteType, serviceType = self.__serviceType,
                resourceType = self.__resourceType, useNewRes = self.useNewRes)

    #DB
    if rsDBIn is not None:
      rsDB = rsDBIn
    else:
      # Use standard DIRAC DB
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      rsDB = ResourceStatusDB()

    if rmDBIn is not None:
      rmDB = rmDBIn
    else:
      # Use standard DIRAC DB
      from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
      rmDB = ResourceManagementDB()

    #setup
    if setupIn is not None:
      setup = setupIn
    else:
      # get present setup
      setup = getSetup()['Value']

    #notification client
    if ncIn is not None:
      nc = ncIn
    else:
      from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
      nc = NotificationClient()

    #DiracAdmin
    if daIn is not None:
      da = daIn
    else:
      from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
      da = DiracAdmin()

    #CSAPI
    if csAPIIn is not None:
      csAPI = csAPIIn
    else:
      from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
      csAPI = CSAPI()


    # policy decision
    resDecisions = pdp.takeDecision(knownInfo=knownInfo)


#    if self.__name == 'CERN-RAW':
#      print resDecisions


    for res in resDecisions['PolicyCombinedResult']:

      self.__policyType = res['PolicyType']

      #if self.__realBan == False:
      #  continue

      if 'Resource_PolType' in self.__policyType:
        # If token != RS_SVC, we do not update the token, just the LastCheckedTime

        if self.__realBan == False:
          rsDB.setLastMonitoredCheckTime(self.__granularity, self.__name)
        else:
          self._ResourcePolTypeActions(resDecisions, res, rsDB, rmDB)

      if 'Alarm_PolType' in self.__policyType:
        self._AlarmPolTypeActions(res, nc, setup, rsDB)

      if 'RealBan_PolType' in self.__policyType and self.__realBan == True:
        self._RealBanPolTypeActions(res, da, csAPI, setup)

      if 'Collective_PolType' in self.__policyType:
        # do something
        pass
コード例 #25
0
ファイル: Publisher.py プロジェクト: zhangxiaomei/DIRAC
class Publisher:
    """
  Class Publisher is in charge of getting dispersed information, to be published on the web.
  """

    #############################################################################

    def __init__(self,
                 VOExtension,
                 rsDBIn=None,
                 commandCallerIn=None,
                 infoGetterIn=None,
                 WMSAdminIn=None):
        """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

        self.configModule = Utils.voimport(
            "DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

        if rsDBIn is not None:
            self.rsDB = rsDBIn
        else:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            self.rsDB = ResourceStatusDB()

        from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
        self.rmDB = ResourceManagementDB()

        if commandCallerIn is not None:
            self.cc = commandCallerIn
        else:
            from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
            self.cc = CommandCaller()

        if infoGetterIn is not None:
            self.ig = infoGetterIn
        else:
            from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
            self.ig = InfoGetter(VOExtension)

        if WMSAdminIn is not None:
            self.WMSAdmin = WMSAdminIn
        else:
            from DIRAC.Core.DISET.RPCClient import RPCClient
            self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

        self.threadPool = ThreadPool(2, 5)

        self.lockObj = threading.RLock()

        self.infoForPanel_res = {}

#############################################################################

    def getInfo(self, granularity, name, useNewRes=False):
        """
    Standard method to get all the info to be published

    This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
    with 2-5 threads. The threaded method is
    :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`

    :params:
      :attr:`granularity`: string - a ValidRes

      :attr:`name`: string - name of the Validres

      :attr:`useNewRes`: boolean. When set to true, will get new results,
      otherwise it will get cached results (where available).
    """

        if granularity not in ValidRes:
            raise InvalidRes, Utils.where(self, self.getInfo)

        self.infoForPanel_res = {}

        status = None
        formerStatus = None
        siteType = None
        serviceType = None
        resourceType = None

        if granularity in ('Resource', 'Resources'):
            try:
                resourceType = self.rsDB.getMonitoredsList(
                    'Resource', ['ResourceType'], resourceName=name)[0][0]
            except IndexError:
                return "%s does not exist!" % name

        if granularity in ('StorageElement', 'StorageElements'):
            try:
                siteType = self.rsDB.getMonitoredsList(
                    'StorageElement', ['SiteType'],
                    storageElementName=name)[0][0]
            except IndexError:
                return "%s does not exist!" % name

        paramNames = [
            'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status',
            'Reason', 'Description'
        ]

        infoToGet = self.ig.getInfoToApply(('view_info', ),
                                           granularity,
                                           status=status,
                                           formerStatus=formerStatus,
                                           siteType=siteType,
                                           serviceType=serviceType,
                                           resourceType=resourceType,
                                           useNewRes=useNewRes)[0]['Panels']
        infoToGet_res = {}

        recordsList = []

        infosForPolicy = {}

        for panel in infoToGet.keys():

            (granularityForPanel,
             nameForPanel) = self.__getNameForPanel(granularity, name, panel)

            if not self._resExist(granularityForPanel, nameForPanel):
                #        completeInfoForPanel_res = None
                continue

            #take composite RSS result for name
            nameStatus_res = self._getStatus(nameForPanel, panel)

            recordBase = [None, None, None, None, None, None, None, None]

            recordBase[1] = panel.replace('_Panel', '')
            recordBase[2] = nameForPanel  #nameForPanel
            try:
                recordBase[4] = nameStatus_res[nameForPanel][
                    'DIRACStatus']  #DIRAC Status
            except:
                pass
            recordBase[5] = nameStatus_res[nameForPanel][
                'RSSStatus']  #RSS Status

            record = copy.deepcopy(recordBase)
            record[0] = 'ResultsForResource'

            recordsList.append(record)

            #take info that goes into the panel
            infoForPanel = infoToGet[panel]

            for info in infoForPanel:

                self.threadPool.generateJobAndQueueIt(
                    self.getInfoForPanel,
                    args=(info, granularityForPanel, nameForPanel))

            self.threadPool.processAllResults()

            for policy in [x.keys()[0] for x in infoForPanel]:
                record = copy.deepcopy(recordBase)
                record[0] = 'SpecificInformation'
                record[3] = policy  #policyName
                record[4] = None  #DIRAC Status
                record[5] = self.infoForPanel_res[policy][
                    'Status']  #RSS status for the policy
                record[6] = self.infoForPanel_res[policy]['Reason']  #Reason
                record[7] = self.infoForPanel_res[policy]['desc']  #Description
                recordsList.append(record)

                infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']

        infoToGet_res['TotalRecords'] = len(recordsList)
        infoToGet_res['ParameterNames'] = paramNames
        infoToGet_res['Records'] = recordsList

        infoToGet_res['Extras'] = infosForPolicy

        return infoToGet_res

#############################################################################

    def getInfoForPanel(self, info, granularityForPanel, nameForPanel):

        #get single RSS policy results
        policyResToGet = info.keys()[0]
        pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet)
        if pol_res != []:
            pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]}
        else:
            pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'}
        self.lockObj.acquire()
        try:
            self.infoForPanel_res[policyResToGet] = pol_res_dict
        finally:
            self.lockObj.release()

        #get policy description
        desc = self._getPolicyDesc(policyResToGet)

        #get other info
        othersInfo = info.values()[0]
        if not isinstance(othersInfo, list):
            othersInfo = [othersInfo]

        info_res = {}

        for oi in othersInfo:
            format_ = oi.keys()[0]
            what = oi.values()[0]

            info_bit_got = self._getInfo(granularityForPanel, nameForPanel,
                                         format_, what)

            info_res[format_] = info_bit_got

        self.lockObj.acquire()
        try:
            self.infoForPanel_res[policyResToGet]['infos'] = info_res
            self.infoForPanel_res[policyResToGet]['desc'] = desc
        finally:
            self.lockObj.release()

#############################################################################

    def _getStatus(self, name, panel):

        #get RSS status
        RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]

        #get DIRAC status
        if panel in ('Site_Panel', 'SE_Panel'):

            if panel == 'Site_Panel':
                DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
                if DIRACStatus['OK']:
                    DIRACStatus = DIRACStatus['Value'][name].pop()[0]
                else:
                    raise RSSException, Utils.where(self, self._getStatus)

            elif panel == 'SE_Panel':
                ra = getStorageElementStatus(name, 'ReadAccess')['Value']
                wa = getStorageElementStatus(name, 'WriteAccess')['Value']
                DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}

            status = {
                name: {
                    'RSSStatus': RSSStatus,
                    'DIRACStatus': DIRACStatus
                }
            }

        else:
            status = {name: {'RSSStatus': RSSStatus}}

        return status

#############################################################################

    def _getInfo(self, granularity, name, format_, what):

        if format_ == 'RSS':
            info_bit_got = self._getInfoFromRSSDB(name, what)
        else:
            if isinstance(what, dict):
                command = what['CommandIn']
                extraArgs = what['args']
            else:
                command = what
                extraArgs = None

            info_bit_got = self.cc.commandInvocation(granularity, name, None,
                                                     None, command, extraArgs)

            try:
                info_bit_got = info_bit_got['Result']
            except:
                pass

        return info_bit_got

#############################################################################

    def _getInfoFromRSSDB(self, name, what):

        paramsL = ['Status']

        siteName = None
        serviceName = None
        resourceName = None
        storageElementName = None
        serviceType = None
        gridSiteName = None

        if what == 'ServiceOfSite':
            gran = 'Service'
            paramsL.insert(0, 'ServiceName')
            paramsL.append('Reason')
            siteName = name
        elif what == 'ResOfCompService':
            gran = 'Resources'
            paramsL.insert(0, 'ResourceName')
            paramsL.append('Reason')
            serviceType = name.split('@')[0]
            gridSiteName = getGOCSiteName(name.split('@')[1])
            if not gridSiteName['OK']:
                raise RSSException, gridSiteName['Message']
            gridSiteName = gridSiteName['Value']
        elif what == 'ResOfStorService':
            gran = 'Resources'
            paramsL.insert(0, 'ResourceName')
            paramsL.append('Reason')
            serviceType = name.split('@')[0]
            gridSiteName = getGOCSiteName(name.split('@')[1])
            if not gridSiteName['OK']:
                raise RSSException, gridSiteName['Message']
            gridSiteName = gridSiteName['Value']
        elif what == 'ResOfStorEl':
            gran = 'StorageElements'
            paramsL.insert(0, 'ResourceName')
            paramsL.append('Reason')
            storageElementName = name
        elif what == 'StorageElementsOfSite':
            gran = 'StorageElements'
            paramsL.insert(0, 'StorageElementName')
            paramsL.append('Reason')
            if '@' in name:
                DIRACsiteName = name.split('@').pop()
            else:
                DIRACsiteName = name
            gridSiteName = getGOCSiteName(DIRACsiteName)
            if not gridSiteName['OK']:
                raise RSSException, gridSiteName['Message']
            gridSiteName = gridSiteName['Value']
        elif what == 'Site_Panel':
            gran = 'Site'
            paramsL.insert(0, 'SiteName')
            siteName = name
        elif what == 'Service_Computing_Panel':
            gran = 'Service'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Service_Storage_Panel':
            gran = 'Service'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Service_VO-BOX_Panel':
            gran = 'Services'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Service_VOMS_Panel':
            gran = 'Services'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Resource_Panel':
            gran = 'Resource'
            paramsL.insert(0, 'ResourceName')
            resourceName = name
        elif what == 'SE_Panel':
            gran = 'StorageElement'
            paramsL.insert(0, 'StorageElementName')
            storageElementName = name

        info_bit_got = self.rsDB.getMonitoredsList(
            gran,
            paramsList=paramsL,
            siteName=siteName,
            serviceName=serviceName,
            serviceType=serviceType,
            resourceName=resourceName,
            storageElementName=storageElementName,
            gridSiteName=gridSiteName)

        return info_bit_got

#############################################################################

    def _getPolicyDesc(self, policyName):

        return self.configModule.Policies[policyName]['Description']

#############################################################################

    def __getNameForPanel(self, granularity, name, panel):

        if granularity in ('Site', 'Sites'):
            if panel == 'Service_Computing_Panel':
                granularity = 'Service'
                name = 'Computing@' + name
            elif panel == 'Service_Storage_Panel':
                granularity = 'Service'
                name = 'Storage@' + name
            elif panel == 'OtherServices_Panel':
                granularity = 'Service'
                name = 'OtherS@' + name
            elif panel == 'Service_VOMS_Panel':
                granularity = 'Service'
                name = 'VOMS@' + name
            elif panel == 'Service_VO-BOX_Panel':
                granularity = 'Service'
                name = 'VO-BOX@' + name
#      else:
#        granularity = granularity
#        name = name
#    else:
#      granularity = granularity
#      name = name

        return (granularity, name)

#############################################################################

    def _resExist(self, granularity, name):

        siteName = None
        serviceName = None
        resourceName = None
        storageElementName = None

        if granularity in ('Site', 'Sites'):
            siteName = name
        elif granularity in ('Service', 'Services'):
            serviceName = name
        elif granularity in ('Resource', 'Resources'):
            resourceName = name
        elif granularity in ('StorageElement', 'StorageElements'):
            storageElementName = name

        res = self.rsDB.getMonitoredsList(
            granularity,
            siteName=siteName,
            serviceName=serviceName,
            resourceName=resourceName,
            storageElementName=storageElementName)

        if res == []:
            return False
        else:
            return True
コード例 #26
0
ファイル: Synchronizer.py プロジェクト: zhangxiaomei/DIRAC
class Synchronizer:

    #############################################################################

    def __init__(self, rsDBin=None, rmDBin=None):

        self.rsDB = rsDBin
        self.rmDB = rmDBin

        if self.rsDB == None and self.rmDB == None:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()

        self.GOCDBClient = GOCDBClient()

#############################################################################

#  def sync(self, thingsToSync = None, fake_param = None):

    def sync(self, _a, _b):
        """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """

        thingsToSync = [
            'Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements',
            'RegistryUsers'
        ]

        gLogger.info("!!! Sync DB content with CS content for %s !!!" %
                     (' '.join(x for x in thingsToSync)))

        for thing in thingsToSync:
            getattr(self, '_sync' + thing)()

        return S_OK()

#############################################################################

    def _syncUtils(self):
        """
    Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
    """

        statusIn = self.rsDB.getStatusList()
        #delete status not more in Utils
        for stIn in statusIn:
            if stIn not in ValidStatus:
                self.rsDB.removeStatus(stIn)
        #Add new status
        for s in ValidStatus:
            if s not in statusIn:
                self.rsDB.addStatus(s)

        for g in ('Site', 'Service', 'Resource'):
            typeIn = self.rsDB.getTypesList(g)
            if g == 'Site':
                typesList = ValidSiteType
            elif g == 'Service':
                typesList = ValidServiceType
            if g == 'Resource':
                typesList = ValidResourceType
            #delete types not more in Utils
            for tIn in typeIn:
                if tIn not in typesList:
                    self.rsDB.removeType(g, tIn)
            #Add new types
            for t in typesList:
                if t not in typeIn:
                    self.rsDB.addType(g, t)

#############################################################################

    def _syncSites(self):
        """
    Sync DB content with sites that are in the CS
    """

        # sites in the DB now
        sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName'])
        sitesIn = [s[0] for s in sitesIn]

        # sites in CS now
        sitesList = getSites()['Value']

        try:
            sitesList.remove('LCG.Dummy.ch')
        except ValueError:
            pass

        # remove sites from the DB not more in the CS
        for site in sitesIn:
            if site not in sitesList:
                self.rsDB.removeSite(site)

        # add to DB what is in CS now and wasn't before
        for site in sitesList:
            if site not in sitesIn:
                # DIRAC Tier
                tier = getSiteTier(site)['Value'][0]
                if tier == 0 or tier == '0':
                    t = 'T0'
                elif tier == 1 or tier == '1':
                    t = 'T1'
                elif tier == 3 or tier == '3':
                    t = 'T3'
                else:
                    t = 'T2'

                #Grid Name of the site
                gridSiteName = getGOCSiteName(site)
                if not gridSiteName['OK']:
                    raise RSSException, gridSiteName['Message']
                gridSiteName = gridSiteName['Value']

                #Grid Tier (with a workaround!)
                DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
                if not DIRACSitesOfGridSites['OK']:
                    raise RSSException, DIRACSitesOfGridSites['Message']
                DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
                if len(DIRACSitesOfGridSites) == 1:
                    gt = t
                else:
                    gt = self.__getGOCTier(DIRACSitesOfGridSites)

                self.rsDB.addOrModifySite(
                    site, t, gridSiteName, 'Active', 'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))

                self.rsDB.addOrModifyGridSite(gridSiteName, gt)

                sitesIn.append(site)

#############################################################################

    def _syncVOBOX(self):
        """
    Sync DB content with VOBoxes
    """

        # services in the DB now
        servicesIn = self.rsDB.getMonitoredsList('Service',
                                                 paramsList=['ServiceName'])
        servicesIn = [s[0] for s in servicesIn]

        for site in [
                'LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk',
                'LCG.GRIDKA.de', 'LCG.NIKHEF.nl'
        ]:
            service = 'VO-BOX@' + site
            if service not in servicesIn:
                self.rsDB.addOrModifyService(
                    service, 'VO-BOX', site, 'Active', 'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))

#############################################################################

    def _syncResources(self):

        # resources in the DB now
        resourcesIn = self.rsDB.getMonitoredsList('Resource',
                                                  paramsList=['ResourceName'])
        resourcesIn = [r[0] for r in resourcesIn]

        # services in the DB now
        servicesIn = self.rsDB.getMonitoredsList('Service',
                                                 paramsList=['ServiceName'])
        servicesIn = [s[0] for s in servicesIn]

        # Site-CE mapping in CS now
        siteCE = getSiteCEMapping('LCG')['Value']
        # Site-SE mapping in CS now
        siteSE = getSiteSEMapping('LCG')['Value']

        # CEs in CS now
        CEList = []
        for i in siteCE.values():
            for ce in i:
                if ce is None:
                    continue
                CEList.append(ce)

        # SEs in CS now
        SEList = []
        for i in siteSE.values():
            for x in i:
                SEList.append(x)

        # SE Nodes in CS now
        SENodeList = []
        for SE in SEList:
            node = getSENodes(SE)['Value'][0]
            if node is None:
                continue
            if node not in SENodeList:
                SENodeList.append(node)

        # LFC Nodes in CS now
        LFCNodeList_L = []
        LFCNodeList_C = []
        for site in getLFCSites()['Value']:
            for readable in ('ReadOnly', 'ReadWrite'):
                LFCNode = getLFCNode(site, readable)['Value']
                if LFCNode is None or LFCNode == []:
                    continue
                LFCNode = LFCNode[0]
                if readable == 'ReadWrite':
                    if LFCNode not in LFCNodeList_C:
                        LFCNodeList_C.append(LFCNode)
                elif readable == 'ReadOnly':
                    if LFCNode not in LFCNodeList_L:
                        LFCNodeList_L.append(LFCNode)

        # FTS Nodes in CS now
        FTSNodeList = []
        sitesWithFTS = getFTSSites()
        for site in sitesWithFTS['Value']:
            fts = getFTSEndpoint(site)['Value']
            if fts is None or fts == []:
                continue
            fts = fts[0]
            if fts not in FTSNodeList:
                FTSNodeList.append(fts)

        # VOMS Nodes in CS now
        VOMSNodeList = getVOMSEndpoints()['Value']

        # complete list of resources in CS now
        resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList

        # list of services in CS now (to be done)
        servicesList = []

        #remove resources no more in the CS
        for res in resourcesIn:
            if res not in resourcesList:
                self.rsDB.removeResource(res)
                self.rsDB.removeStorageElement(resourceName=res)

        # add to DB what is in CS now and wasn't before

        # CEs
        for site in siteCE.keys():
            if site == 'LCG.Dummy.ch':
                continue
            for ce in siteCE[site]:
                if ce is None:
                    continue
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', ce)
                if not siteInGOCDB['OK']:
                    raise RSSException, siteInGOCDB['Message']
                if siteInGOCDB['Value'] == []:
                    trueName = socket.gethostbyname_ex(ce)[0]
                    siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                        'hostname', trueName)
                try:
                    siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
                except IndexError:
                    continue
                serviceType = 'Computing'
                service = serviceType + '@' + site

                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)

                if ce not in resourcesIn:
                    CEType = getCEType(site, ce)['Value']
                    ceType = 'CE'
                    if CEType == 'CREAM':
                        ceType = 'CREAMCE'
                    self.rsDB.addOrModifyResource(
                        ce, ceType, serviceType, site, siteInGOCDB, 'Active',
                        'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    resourcesIn.append(ce)

        # SRMs
        for srm in SENodeList:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', srm)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(srm)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)

            if srm not in resourcesIn and srm is not None:
                self.rsDB.addOrModifyResource(
                    srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(srm)

        # LFC_C
        for lfc in LFCNodeList_C:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', lfc)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(lfc)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if lfc not in resourcesIn and lfc is not None:
                self.rsDB.addOrModifyResource(
                    lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(lfc)

        # LFC_L
        for lfc in LFCNodeList_L:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', lfc)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(lfc)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if lfc not in resourcesIn and lfc is not None:
                self.rsDB.addOrModifyResource(
                    lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(lfc)

        # FTSs
        for fts in FTSNodeList:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', fts)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(fts)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if fts not in resourcesIn and fts is not None:
                self.rsDB.addOrModifyResource(
                    fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(fts)

        # VOMSs
        for voms in VOMSNodeList:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', voms)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(voms)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            site = siteInDIRAC['Value']
            serviceType = 'VOMS'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if voms not in resourcesIn and voms is not None:
                self.rsDB.addOrModifyResource(
                    voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(voms)

        #remove services no more in the CS
        for ser in servicesIn:
            if ser not in servicesList:
                serType = ser.split('@')[0]
                if serType != 'VO-BOX':
                    self.rsDB.removeService(ser)
                    self.rsDB.removeResource(serviceName=ser)
                    site = ser.split('@')[1]
                    if serType == 'Storage':
                        self.rsDB.removeStorageElement(siteName=site)

#############################################################################

    def _syncStorageElements(self):

        # Get StorageElements from the CS
        SEs = getStorageElements()
        if not SEs['OK']:
            raise RSSException, SEs['Message']
        SEs = SEs['Value']

        for access in ('Read', 'Write'):

            storageElementsIn = self.rsDB.getMonitoredsList(
                'StorageElement' + access, paramsList=['StorageElementName'])
            try:
                storageElementsIn = [x[0] for x in storageElementsIn]
            except IndexError:
                pass

            #remove storageElements no more in the CS
            for se in storageElementsIn:
                if se not in SEs:
                    self.rsDB.removeStorageElement(storageElementName=se,
                                                   resourceName=None,
                                                   access=access)

            #Add new storage Elements
            for SE in SEs:
                srm = getSENodes(SE)['Value'][0]
                if srm == None:
                    continue
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', srm)
                if not siteInGOCDB['OK']:
                    raise RSSException, siteInGOCDB['Message']
                if siteInGOCDB['Value'] == []:
                    continue
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']

                if SE not in storageElementsIn:
                    self.rsDB.addOrModifyStorageElement(
                        SE,
                        srm,
                        siteInGOCDB,
                        'Active',
                        'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC',
                        datetime.datetime(9999, 12, 31, 23, 59, 59),
                        access=access)
                    storageElementsIn.append(SE)

#############################################################################

    def __getGOCTier(self, sitesList):

        gridTier = 3

        for site in sitesList:

            tier = getSiteTier(site)['Value'][0]

            if tier == 0 or tier == '0':
                tn = 0
            elif tier == 1 or tier == '1':
                tn = 1
            elif tier == 3 or tier == '3':
                tn = 3
            else:
                tn = 2

            if tn < gridTier:
                gridTier = tn

        if gridTier == 0:
            gt = 'T0'
        elif gridTier == 1:
            gt = 'T1'
        elif gridTier == 3:
            gt = 'T3'
        else:
            gt = 'T2'

        return gt


#############################################################################

    def _syncRegistryUsers(self):
        from DIRAC.ResourceStatusSystem.Utilities import CS
        users = CS.getTypedDictRootedAt("Users", root="/Registry")
        for u in users:
            if type(users[u]['DN']) == list:
                users[u]['DN'] = users[u]['DN'][0]
            if type(users[u]['Email']) == list:
                users[u]['Email'] = users[u]['Email'][0]

            users[u]['DN'] = users[u]['DN'].split('=')[-1]
            self.rmDB.registryAddUser(u, users[u]['DN'].lower(),
                                      users[u]['Email'].lower())