Esempio n. 1
0
class Publisher:
  """
  Class Publisher is in charge of getting dispersed information, to be published on the web.
  """

#############################################################################

  def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
               WMSAdminIn = None):
    """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

    self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

    if rsDBIn is not None:
      self.rsDB = rsDBIn
    else:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      self.rsDB = ResourceStatusDB()

    from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
    self.rmDB = ResourceManagementDB()

    if commandCallerIn is not None:
      self.cc = commandCallerIn
    else:
      from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
      self.cc = CommandCaller()

    if infoGetterIn is not None:
      self.ig = infoGetterIn
    else:
      from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
      self.ig = InfoGetter(VOExtension)

    if WMSAdminIn is not None:
      self.WMSAdmin = WMSAdminIn
    else:
      from DIRAC.Core.DISET.RPCClient import RPCClient
      self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

    self.threadPool = ThreadPool( 2, 5 )

    self.lockObj = threading.RLock()

    self.infoForPanel_res = {}

#############################################################################

  def getInfo(self, granularity, name, useNewRes = False):
    """
    Standard method to get all the info to be published

    This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
    with 2-5 threads. The threaded method is
    :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`

    :params:
      :attr:`granularity`: string - a ValidRes

      :attr:`name`: string - name of the Validres

      :attr:`useNewRes`: boolean. When set to true, will get new results,
      otherwise it will get cached results (where available).
    """

    if granularity not in ValidRes:
      raise InvalidRes, Utils.where(self, self.getInfo)

    self.infoForPanel_res = {}

    status = None
    formerStatus = None
    siteType = None
    serviceType = None
    resourceType = None

    if granularity in ('Resource', 'Resources'):
      try:
        resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'],
                                              resourceName = name)[0][0]
      except IndexError:
        return "%s does not exist!" %name

    if granularity in ('StorageElement', 'StorageElements'):
      try:
        siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'],
                                              storageElementName = name)[0][0]
      except IndexError:
        return "%s does not exist!" %name

    paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status',
                  'RSS Status', 'Reason', 'Description']

    infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status,
                                       formerStatus = formerStatus, siteType = siteType,
                                       serviceType = serviceType, resourceType = resourceType,
                                       useNewRes = useNewRes)[0]['Panels']
    infoToGet_res = {}

    recordsList = []

    infosForPolicy = {}

    for panel in infoToGet.keys():

      (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel)

      if not self._resExist(granularityForPanel, nameForPanel):
#        completeInfoForPanel_res = None
        continue

      #take composite RSS result for name
      nameStatus_res = self._getStatus(nameForPanel, panel)

      recordBase = [None, None, None, None, None, None, None, None]

      recordBase[1] = panel.replace('_Panel', '')
      recordBase[2] = nameForPanel #nameForPanel
      try:
        recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status
      except:
        pass
      recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status

      record = copy.deepcopy(recordBase)
      record[0] = 'ResultsForResource'

      recordsList.append(record)

      #take info that goes into the panel
      infoForPanel = infoToGet[panel]

      for info in infoForPanel:

        self.threadPool.generateJobAndQueueIt(self.getInfoForPanel,
                                              args = (info, granularityForPanel, nameForPanel) )

      self.threadPool.processAllResults()

      for policy in [x.keys()[0] for x in infoForPanel]:
        record = copy.deepcopy(recordBase)
        record[0] = 'SpecificInformation'
        record[3] = policy #policyName
        record[4] = None #DIRAC Status
        record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy
        record[6] = self.infoForPanel_res[policy]['Reason'] #Reason
        record[7] = self.infoForPanel_res[policy]['desc'] #Description
        recordsList.append(record)

        infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']

    infoToGet_res['TotalRecords'] = len(recordsList)
    infoToGet_res['ParameterNames'] = paramNames
    infoToGet_res['Records'] = recordsList

    infoToGet_res['Extras'] = infosForPolicy

    return infoToGet_res

#############################################################################

  def getInfoForPanel(self, info, granularityForPanel, nameForPanel):

    #get single RSS policy results
    policyResToGet = info.keys()[0]
    pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet)
    if pol_res != []:
      pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]}
    else:
      pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'}
    self.lockObj.acquire()
    try:
      self.infoForPanel_res[policyResToGet] = pol_res_dict
    finally:
      self.lockObj.release()

    #get policy description
    desc = self._getPolicyDesc(policyResToGet)

    #get other info
    othersInfo = info.values()[0]
    if not isinstance(othersInfo, list):
      othersInfo = [othersInfo]

    info_res = {}

    for oi in othersInfo:
      format_ = oi.keys()[0]
      what = oi.values()[0]

      info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what)

      info_res[format_] = info_bit_got

    self.lockObj.acquire()
    try:
      self.infoForPanel_res[policyResToGet]['infos'] = info_res
      self.infoForPanel_res[policyResToGet]['desc'] = desc
    finally:
      self.lockObj.release()

#############################################################################

  def _getStatus(self, name, panel):

    #get RSS status
    RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]

    #get DIRAC status
    if panel in ('Site_Panel', 'SE_Panel'):

      if panel == 'Site_Panel':
        DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
        if DIRACStatus['OK']:
          DIRACStatus = DIRACStatus['Value'][name].pop()[0]
        else:
          raise RSSException, Utils.where(self, self._getStatus)

      elif panel == 'SE_Panel':
        ra = getStorageElementStatus(name, 'ReadAccess')['Value']
        wa = getStorageElementStatus(name, 'WriteAccess')['Value']
        DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}

      status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } }

    else:
      status = { name : { 'RSSStatus': RSSStatus} }


    return status

#############################################################################

  def _getInfo(self, granularity, name, format_, what):

    if format_ == 'RSS':
      info_bit_got = self._getInfoFromRSSDB(name, what)
    else:
      if isinstance(what, dict):
        command = what['CommandIn']
        extraArgs = what['args']
      else:
        command = what
        extraArgs = None

      info_bit_got = self.cc.commandInvocation(granularity, name, None,
                                               None, command, extraArgs)

      try:
        info_bit_got = info_bit_got['Result']
      except:
        pass

    return info_bit_got

#############################################################################

  def _getInfoFromRSSDB(self, name, what):

    paramsL = ['Status']

    siteName = None
    serviceName = None
    resourceName = None
    storageElementName = None
    serviceType = None
    gridSiteName = None

    if what == 'ServiceOfSite':
      gran = 'Service'
      paramsL.insert(0, 'ServiceName')
      paramsL.append('Reason')
      siteName = name
    elif what == 'ResOfCompService':
      gran = 'Resources'
      paramsL.insert(0, 'ResourceName')
      paramsL.append('Reason')
      serviceType = name.split('@')[0]
      gridSiteName = getGOCSiteName(name.split('@')[1])
      if not gridSiteName['OK']:
        raise RSSException, gridSiteName['Message']
      gridSiteName = gridSiteName['Value']
    elif what == 'ResOfStorService':
      gran = 'Resources'
      paramsL.insert(0, 'ResourceName')
      paramsL.append('Reason')
      serviceType = name.split('@')[0]
      gridSiteName = getGOCSiteName(name.split('@')[1])
      if not gridSiteName['OK']:
        raise RSSException, gridSiteName['Message']
      gridSiteName = gridSiteName['Value']
    elif what == 'ResOfStorEl':
      gran = 'StorageElements'
      paramsL.insert(0, 'ResourceName')
      paramsL.append('Reason')
      storageElementName = name
    elif what == 'StorageElementsOfSite':
      gran = 'StorageElements'
      paramsL.insert(0, 'StorageElementName')
      paramsL.append('Reason')
      if '@' in name:
        DIRACsiteName = name.split('@').pop()
      else:
        DIRACsiteName = name
      gridSiteName = getGOCSiteName(DIRACsiteName)
      if not gridSiteName['OK']:
        raise RSSException, gridSiteName['Message']
      gridSiteName = gridSiteName['Value']
    elif what == 'Site_Panel':
      gran = 'Site'
      paramsL.insert(0, 'SiteName')
      siteName = name
    elif what == 'Service_Computing_Panel':
      gran = 'Service'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Service_Storage_Panel':
      gran = 'Service'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Service_VO-BOX_Panel':
      gran = 'Services'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Service_VOMS_Panel':
      gran = 'Services'
      paramsL.insert(0, 'ServiceName')
      serviceName = name
    elif what == 'Resource_Panel':
      gran = 'Resource'
      paramsL.insert(0, 'ResourceName')
      resourceName = name
    elif what == 'SE_Panel':
      gran = 'StorageElement'
      paramsL.insert(0, 'StorageElementName')
      storageElementName = name

    info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName,
                                               serviceName = serviceName, serviceType = serviceType,
                                               resourceName = resourceName,
                                               storageElementName = storageElementName,
                                               gridSiteName = gridSiteName)

    return info_bit_got

#############################################################################

  def _getPolicyDesc(self, policyName):

    return self.configModule.Policies[policyName]['Description']

#############################################################################

  def __getNameForPanel(self, granularity, name, panel):

    if granularity in ('Site', 'Sites'):
      if panel == 'Service_Computing_Panel':
        granularity = 'Service'
        name = 'Computing@' + name
      elif panel == 'Service_Storage_Panel':
        granularity = 'Service'
        name = 'Storage@' + name
      elif panel == 'OtherServices_Panel':
        granularity = 'Service'
        name = 'OtherS@' + name
      elif panel == 'Service_VOMS_Panel':
        granularity = 'Service'
        name = 'VOMS@' + name
      elif panel == 'Service_VO-BOX_Panel':
        granularity = 'Service'
        name = 'VO-BOX@' + name
#      else:
#        granularity = granularity
#        name = name
#    else:
#      granularity = granularity
#      name = name

    return (granularity, name)

#############################################################################

  def _resExist(self, granularity, name):

    siteName = None
    serviceName = None
    resourceName = None
    storageElementName = None

    if granularity in ('Site', 'Sites'):
      siteName = name
    elif granularity in ('Service', 'Services'):
      serviceName = name
    elif granularity in ('Resource', 'Resources'):
      resourceName = name
    elif granularity in ('StorageElement', 'StorageElements'):
      storageElementName = name

    res = self.rsDB.getMonitoredsList(granularity, siteName = siteName,
                                      serviceName = serviceName, resourceName = resourceName,
                                      storageElementName = storageElementName)

    if res == []:
      return False
    else:
      return True
Esempio n. 2
0
class Synchronizer:

#############################################################################

  def __init__( self, rsDBin = None, rmDBin = None ):

    self.rsDB = rsDBin
    self.rmDB = rmDBin

    if self.rsDB == None and self.rmDB == None:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

    self.GOCDBClient = GOCDBClient()

#############################################################################

#  def sync(self, thingsToSync = None, fake_param = None):
  def sync( self, _a, _b ):
    """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """

    thingsToSync = ['Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers']

    gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ' '.join( x for x in thingsToSync ) ) )

    for thing in thingsToSync:
      getattr( self, '_sync' + thing )()

    return S_OK()

#############################################################################

  def _syncUtils( self ):
    """
    Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
    """

    statusIn = self.rsDB.getStatusList()
    #delete status not more in Utils
    for stIn in statusIn:
      if stIn not in ValidStatus:
        self.rsDB.removeStatus( stIn )
    #Add new status
    for s in ValidStatus:
      if s not in statusIn:
        self.rsDB.addStatus( s )

    for g in ( 'Site', 'Service', 'Resource' ):
      typeIn = self.rsDB.getTypesList( g )
      if g == 'Site':
        typesList = ValidSiteType
      elif g == 'Service':
        typesList = ValidServiceType
      if g == 'Resource':
        typesList = ValidResourceType
      #delete types not more in Utils
      for tIn in typeIn:
        if tIn not in typesList:
          self.rsDB.removeType( g, tIn )
      #Add new types
      for t in typesList:
        if t not in typeIn:
          self.rsDB.addType( g, t )

#############################################################################

  def _syncSites( self ):
    """
    Sync DB content with sites that are in the CS
    """

    # sites in the DB now
    sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] )
    sitesIn = [s[0] for s in sitesIn]

    # sites in CS now
    sitesList = getSites()['Value']

    try:
      sitesList.remove( 'LCG.Dummy.ch' )
    except ValueError:
      pass

    # remove sites from the DB not more in the CS
    for site in sitesIn:
      if site not in sitesList:
        self.rsDB.removeSite( site )

    # add to DB what is in CS now and wasn't before
    for site in sitesList:
      if site not in sitesIn:
        # DIRAC Tier
        tier = getSiteTier( site )['Value'][0]
        if tier == 0 or tier == '0':
          t = 'T0'
        elif tier == 1 or tier == '1':
          t = 'T1'
        elif tier == 3 or tier == '3':
          t = 'T3'
        else:
          t = 'T2'

        #Grid Name of the site
        gridSiteName = getGOCSiteName( site )
        if not gridSiteName['OK']:
          raise RSSException, gridSiteName['Message']
        gridSiteName = gridSiteName['Value']

        #Grid Tier (with a workaround!)
        DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
        if not DIRACSitesOfGridSites['OK']:
          raise RSSException, DIRACSitesOfGridSites['Message']
        DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
        if len( DIRACSitesOfGridSites ) == 1:
          gt = t
        else:
          gt = self.__getGOCTier( DIRACSitesOfGridSites )

        self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init',
                                  datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                  datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )

        self.rsDB.addOrModifyGridSite( gridSiteName, gt )

        sitesIn.append( site )

#############################################################################

  def _syncVOBOX( self ):
    """
    Sync DB content with VOBoxes
    """

    # services in the DB now
    servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
    servicesIn = [s[0] for s in servicesIn]

    for site in ['LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es',
                 'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl']:
      service = 'VO-BOX@' + site
      if service not in servicesIn:
        self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init',
                                     datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                     datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )


#############################################################################

  def _syncResources( self ):

    # resources in the DB now
    resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] )
    resourcesIn = [r[0] for r in resourcesIn]

    # services in the DB now
    servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
    servicesIn = [s[0] for s in servicesIn]

    # Site-CE mapping in CS now
    siteCE = getSiteCEMapping( 'LCG' )['Value']
    # Site-SE mapping in CS now
    siteSE = getSiteSEMapping( 'LCG' )['Value']

    # CEs in CS now
    CEList = []
    for i in siteCE.values():
      for ce in i:
        if ce is None:
          continue
        CEList.append( ce )

    # SEs in CS now
    SEList = []
    for i in siteSE.values():
      for x in i:
        SEList.append( x )

    # SE Nodes in CS now
    SENodeList = []
    for SE in SEList:
      node = getSENodes( SE )['Value'][0]
      if node is None:
        continue
      if node not in SENodeList:
        SENodeList.append( node )

    # LFC Nodes in CS now
    LFCNodeList_L = []
    LFCNodeList_C = []
    for site in getLFCSites()['Value']:
      for readable in ( 'ReadOnly', 'ReadWrite' ):
        LFCNode = getLFCNode( site, readable )['Value']
        if LFCNode is None or LFCNode == []:
          continue
        LFCNode = LFCNode[0]
        if readable == 'ReadWrite':
          if LFCNode not in LFCNodeList_C:
            LFCNodeList_C.append( LFCNode )
        elif readable == 'ReadOnly':
          if LFCNode not in LFCNodeList_L:
            LFCNodeList_L.append( LFCNode )

    # FTS Nodes in CS now
    FTSNodeList = []
    sitesWithFTS = getFTSSites()
    for site in sitesWithFTS['Value']:
      fts = getFTSEndpoint( site )['Value']
      if fts is None or fts == []:
        continue
      fts = fts[0]
      if fts not in FTSNodeList:
        FTSNodeList.append( fts )

    # VOMS Nodes in CS now
    VOMSNodeList = getVOMSEndpoints()['Value']


    # complete list of resources in CS now
    resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList

    # list of services in CS now (to be done)
    servicesList = []

    #remove resources no more in the CS
    for res in resourcesIn:
      if res not in resourcesList:
        self.rsDB.removeResource( res )
        self.rsDB.removeStorageElement( resourceName = res )


    # add to DB what is in CS now and wasn't before

    # CEs
    for site in siteCE.keys():
      if site == 'LCG.Dummy.ch':
        continue
      for ce in siteCE[site]:
        if ce is None:
          continue
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce )
        if not siteInGOCDB['OK']:
          raise RSSException, siteInGOCDB['Message']
        if siteInGOCDB['Value'] == []:
          trueName = socket.gethostbyname_ex( ce )[0]
          siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
        try:
          siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
        except IndexError:
          continue
        serviceType = 'Computing'
        service = serviceType + '@' + site

        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )

        if ce not in resourcesIn:
          CEType = getCEType( site, ce )['Value']
          ceType = 'CE'
          if CEType == 'CREAM':
            ceType = 'CREAMCE'
          self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init',
                                        datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                        datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          resourcesIn.append( ce )

    # SRMs
    for srm in SENodeList:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( srm )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )

      if srm not in resourcesIn and srm is not None:
        self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( srm )

    # LFC_C
    for lfc in LFCNodeList_C:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( lfc )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if lfc not in resourcesIn and lfc is not None:
        self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( lfc )

    # LFC_L
    for lfc in LFCNodeList_L:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( lfc )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if lfc not in resourcesIn and lfc is not None:
        self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( lfc )


    # FTSs
    for fts in FTSNodeList:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( fts )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      sites = siteInDIRAC['Value']
      serviceType = 'Storage'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if fts not in resourcesIn and fts is not None:
        self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( fts )

    # VOMSs
    for voms in VOMSNodeList:
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms )
      if not siteInGOCDB['OK']:
        raise RSSException, siteInGOCDB['Message']
      if siteInGOCDB['Value'] == []:
        trueName = socket.gethostbyname_ex( voms )[0]
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
      try:
        siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
      except IndexError:
        continue
      siteInDIRAC = getDIRACSiteName( siteInGOCDB )
      if not siteInDIRAC['OK']:
        raise RSSException, siteInDIRAC['Message']
      site = siteInDIRAC['Value']
      serviceType = 'VOMS'
      for site in sites:
        service = serviceType + '@' + site
        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )
      if voms not in resourcesIn and voms is not None:
        self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
                                      datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                      datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
        resourcesIn.append( voms )

    #remove services no more in the CS
    for ser in servicesIn:
      if ser not in servicesList:
        serType = ser.split( '@' )[0]
        if serType != 'VO-BOX':
          self.rsDB.removeService( ser )
          self.rsDB.removeResource( serviceName = ser )
          site = ser.split( '@' )[1]
          if serType == 'Storage':
            self.rsDB.removeStorageElement( siteName = site )


#############################################################################

  def _syncStorageElements( self ):

    # Get StorageElements from the CS
    SEs = getStorageElements()
    if not SEs['OK']:
      raise RSSException, SEs['Message']
    SEs = SEs['Value']

    for access in ( 'Read', 'Write' ):

      storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access,
                                                       paramsList = [ 'StorageElementName' ] )
      try:
        storageElementsIn = [ x[ 0 ] for x in storageElementsIn ]
      except IndexError:
        pass

      #remove storageElements no more in the CS
      for se in storageElementsIn:
        if se not in SEs:
          self.rsDB.removeStorageElement( storageElementName = se, resourceName = None, access = access )

      #Add new storage Elements
      for SE in SEs:
        srm = getSENodes( SE )[ 'Value' ][ 0 ]
        if srm == None:
          continue
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
        if not siteInGOCDB[ 'OK' ]:
          raise RSSException, siteInGOCDB[ 'Message' ]
        if siteInGOCDB[ 'Value' ] == []:
          continue
        siteInGOCDB = siteInGOCDB[ 'Value' ][ 0 ][ 'SITENAME' ]

        if SE not in storageElementsIn:
          self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init',
                                               datetime.datetime.utcnow().replace( microsecond = 0 ),
                                              'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ),
                                               access = access )
          storageElementsIn.append( SE )

#############################################################################

  def __getGOCTier( self, sitesList ):

    gridTier = 3

    for site in sitesList:

      tier = getSiteTier( site )['Value'][0]

      if tier == 0 or tier == '0':
        tn = 0
      elif tier == 1 or tier == '1':
        tn = 1
      elif tier == 3 or tier == '3':
        tn = 3
      else:
        tn = 2

      if tn < gridTier:
        gridTier = tn

    if gridTier == 0:
      gt = 'T0'
    elif gridTier == 1:
      gt = 'T1'
    elif gridTier == 3:
      gt = 'T3'
    else:
      gt = 'T2'

    return gt

#############################################################################

  def _syncRegistryUsers(self):
    from DIRAC.ResourceStatusSystem.Utilities import CS
    users = CS.getTypedDictRootedAt("Users", root= "/Registry")
    for u in users:
      if type(users[u]['DN']) == list:
        users[u]['DN'] = users[u]['DN'][0]
      if type(users[u]['Email']) == list:
        users[u]['Email'] = users[u]['Email'][0]

      users[u]['DN'] = users[u]['DN'].split('=')[-1]
      self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())
Esempio n. 3
0
class Publisher:
    """
  Class Publisher is in charge of getting dispersed information, to be published on the web.
  """

    #############################################################################

    def __init__(self,
                 VOExtension,
                 rsDBIn=None,
                 commandCallerIn=None,
                 infoGetterIn=None,
                 WMSAdminIn=None):
        """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

        self.configModule = __import__(
            VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
            globals(), locals(), ['*'])

        if rsDBIn is not None:
            self.rsDB = rsDBIn
        else:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            self.rsDB = ResourceStatusDB()

        if commandCallerIn is not None:
            self.cc = commandCallerIn
        else:
            from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
            self.cc = CommandCaller()

        if infoGetterIn is not None:
            self.ig = infoGetterIn
        else:
            from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
            self.ig = InfoGetter(VOExtension)

        if WMSAdminIn is not None:
            self.WMSAdmin = WMSAdminIn
        else:
            from DIRAC.Core.DISET.RPCClient import RPCClient
            self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

        self.threadPool = ThreadPool(2, 5)

        self.lockObj = threading.RLock()

        self.infoForPanel_res = {}

#############################################################################

    def getInfo(self, granularity, name, useNewRes=False):
        """
    Standard method to get all the info to be published

    This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
    with 2-5 threads. The threaded method is
    :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`

    :params:
      :attr:`granularity`: string - a ValidRes

      :attr:`name`: string - name of the Validres

      :attr:`useNewRes`: boolean. When set to true, will get new results,
      otherwise it will get cached results (where available).
    """

        if granularity not in ValidRes:
            raise InvalidRes, where(self, self.getInfo)

        self.infoForPanel_res = {}

        status = None
        formerStatus = None
        siteType = None
        serviceType = None
        resourceType = None

        if granularity in ('Resource', 'Resources'):
            try:
                resourceType = self.rsDB.getMonitoredsList(
                    'Resource', ['ResourceType'], resourceName=name)[0][0]
            except IndexError:
                return "%s does not exist!" % name

        if granularity in ('StorageElement', 'StorageElements'):
            try:
                siteType = self.rsDB.getMonitoredsList(
                    'StorageElement', ['SiteType'],
                    storageElementName=name)[0][0]
            except IndexError:
                return "%s does not exist!" % name

        paramNames = [
            'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status',
            'Reason', 'Description'
        ]

        infoToGet = self.ig.getInfoToApply(('view_info', ),
                                           granularity,
                                           status=status,
                                           formerStatus=formerStatus,
                                           siteType=siteType,
                                           serviceType=serviceType,
                                           resourceType=resourceType,
                                           useNewRes=useNewRes)[0]['Panels']
        infoToGet_res = {}

        recordsList = []

        infosForPolicy = {}

        for panel in infoToGet.keys():

            (granularityForPanel,
             nameForPanel) = self.__getNameForPanel(granularity, name, panel)

            if not self._resExist(granularityForPanel, nameForPanel):
                #        completeInfoForPanel_res = None
                continue

            #take composite RSS result for name
            nameStatus_res = self._getStatus(nameForPanel, panel)

            recordBase = [None, None, None, None, None, None, None, None]

            recordBase[1] = panel.replace('_Panel', '')
            recordBase[2] = nameForPanel  #nameForPanel
            try:
                recordBase[4] = nameStatus_res[nameForPanel][
                    'DIRACStatus']  #DIRAC Status
            except:
                pass
            recordBase[5] = nameStatus_res[nameForPanel][
                'RSSStatus']  #RSS Status

            record = copy.deepcopy(recordBase)
            record[0] = 'ResultsForResource'

            recordsList.append(record)

            #take info that goes into the panel
            infoForPanel = infoToGet[panel]

            for info in infoForPanel:

                self.threadPool.generateJobAndQueueIt(
                    self.getInfoForPanel,
                    args=(info, granularityForPanel, nameForPanel))

            self.threadPool.processAllResults()

            for policy in [x.keys()[0] for x in infoForPanel]:
                record = copy.deepcopy(recordBase)
                record[0] = 'SpecificInformation'
                record[3] = policy  #policyName
                record[4] = None  #DIRAC Status
                record[5] = self.infoForPanel_res[policy][
                    'Status']  #RSS status for the policy
                record[6] = self.infoForPanel_res[policy]['Reason']  #Reason
                record[7] = self.infoForPanel_res[policy]['desc']  #Description
                recordsList.append(record)

                infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']

        infoToGet_res['TotalRecords'] = len(recordsList)
        infoToGet_res['ParameterNames'] = paramNames
        infoToGet_res['Records'] = recordsList

        infoToGet_res['Extras'] = infosForPolicy

        return infoToGet_res

#############################################################################

    def getInfoForPanel(self, info, granularityForPanel, nameForPanel):

        #get single RSS policy results
        policyResToGet = info.keys()[0]
        pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet)
        if pol_res != []:
            pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]}
        else:
            pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'}
        self.lockObj.acquire()
        try:
            self.infoForPanel_res[policyResToGet] = pol_res_dict
        finally:
            self.lockObj.release()

        #get policy description
        desc = self._getPolicyDesc(policyResToGet)

        #get other info
        othersInfo = info.values()[0]
        if not isinstance(othersInfo, list):
            othersInfo = [othersInfo]

        info_res = {}

        for oi in othersInfo:
            format = oi.keys()[0]
            what = oi.values()[0]

            info_bit_got = self._getInfo(granularityForPanel, nameForPanel,
                                         format, what)

            info_res[format] = info_bit_got

        self.lockObj.acquire()
        try:
            self.infoForPanel_res[policyResToGet]['infos'] = info_res
            self.infoForPanel_res[policyResToGet]['desc'] = desc
        finally:
            self.lockObj.release()

#############################################################################

    def _getStatus(self, name, panel):

        #get RSS status
        RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]

        #get DIRAC status
        if panel in ('Site_Panel', 'SE_Panel'):

            if panel == 'Site_Panel':
                DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
                if DIRACStatus['OK']:
                    DIRACStatus = DIRACStatus['Value'][name].pop()[0]
                else:
                    raise RSSException, where(self, self._getStatus)

            elif panel == 'SE_Panel':
                ra = getStorageElementStatus(name, 'ReadAccess')['Value']
                wa = getStorageElementStatus(name, 'WriteAccess')['Value']
                DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}

            status = {
                name: {
                    'RSSStatus': RSSStatus,
                    'DIRACStatus': DIRACStatus
                }
            }

        else:
            status = {name: {'RSSStatus': RSSStatus}}

        return status

#############################################################################

    def _getInfo(self, granularity, name, format, what):

        if format == 'RSS':
            info_bit_got = self._getInfoFromRSSDB(name, what)
        else:
            if isinstance(what, dict):
                command = what['CommandIn']
                extraArgs = what['args']
            else:
                command = what
                extraArgs = None

            info_bit_got = self.cc.commandInvocation(granularity, name, None,
                                                     None, command, extraArgs)

            try:
                info_bit_got = info_bit_got['Result']
            except:
                pass

        return info_bit_got

#############################################################################

    def _getInfoFromRSSDB(self, name, what):

        paramsL = ['Status']

        siteName = None
        serviceName = None
        resourceName = None
        storageElementName = None
        serviceType = None
        gridSiteName = None

        if what == 'ServiceOfSite':
            gran = 'Service'
            paramsL.insert(0, 'ServiceName')
            paramsL.append('Reason')
            siteName = name
        elif what == 'ResOfCompService':
            gran = 'Resources'
            paramsL.insert(0, 'ResourceName')
            paramsL.append('Reason')
            serviceType = name.split('@')[0]
            gridSiteName = getGOCSiteName(name.split('@')[1])
            if not gridSiteName['OK']:
                raise RSSException, gridSiteName['Message']
            gridSiteName = gridSiteName['Value']
        elif what == 'ResOfStorService':
            gran = 'Resources'
            paramsL.insert(0, 'ResourceName')
            paramsL.append('Reason')
            serviceType = name.split('@')[0]
            gridSiteName = getGOCSiteName(name.split('@')[1])
            if not gridSiteName['OK']:
                raise RSSException, gridSiteName['Message']
            gridSiteName = gridSiteName['Value']
        elif what == 'ResOfStorEl':
            gran = 'StorageElements'
            paramsL.insert(0, 'ResourceName')
            paramsL.append('Reason')
            storageElementName = name
        elif what == 'StorageElementsOfSite':
            gran = 'StorageElements'
            paramsL.insert(0, 'StorageElementName')
            paramsL.append('Reason')
            if '@' in name:
                DIRACsiteName = name.split('@').pop()
            else:
                DIRACsiteName = name
            gridSiteName = getGOCSiteName(DIRACsiteName)
            if not gridSiteName['OK']:
                raise RSSException, gridSiteName['Message']
            gridSiteName = gridSiteName['Value']
        elif what == 'Site_Panel':
            gran = 'Site'
            paramsL.insert(0, 'SiteName')
            siteName = name
        elif what == 'Service_Computing_Panel':
            gran = 'Service'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Service_Storage_Panel':
            gran = 'Service'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Service_VO-BOX_Panel':
            gran = 'Services'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Service_VOMS_Panel':
            gran = 'Services'
            paramsL.insert(0, 'ServiceName')
            serviceName = name
        elif what == 'Resource_Panel':
            gran = 'Resource'
            paramsL.insert(0, 'ResourceName')
            resourceName = name
        elif what == 'SE_Panel':
            gran = 'StorageElement'
            paramsL.insert(0, 'StorageElementName')
            storageElementName = name

        info_bit_got = self.rsDB.getMonitoredsList(
            gran,
            paramsList=paramsL,
            siteName=siteName,
            serviceName=serviceName,
            serviceType=serviceType,
            resourceName=resourceName,
            storageElementName=storageElementName,
            gridSiteName=gridSiteName)

        return info_bit_got

#############################################################################

    def _getPolicyDesc(self, policyName):

        return self.configModule.Policies[policyName]['Description']

#############################################################################

    def __getNameForPanel(self, granularity, name, panel):

        if granularity in ('Site', 'Sites'):
            if panel == 'Service_Computing_Panel':
                granularity = 'Service'
                name = 'Computing@' + name
            elif panel == 'Service_Storage_Panel':
                granularity = 'Service'
                name = 'Storage@' + name
            elif panel == 'OtherServices_Panel':
                granularity = 'Service'
                name = 'OtherS@' + name
            elif panel == 'Service_VOMS_Panel':
                granularity = 'Service'
                name = 'VOMS@' + name
            elif panel == 'Service_VO-BOX_Panel':
                granularity = 'Service'
                name = 'VO-BOX@' + name
#      else:
#        granularity = granularity
#        name = name
#    else:
#      granularity = granularity
#      name = name

        return (granularity, name)

#############################################################################

    def _resExist(self, granularity, name):

        siteName = None
        serviceName = None
        resourceName = None
        storageElementName = None

        if granularity in ('Site', 'Sites'):
            siteName = name
        elif granularity in ('Service', 'Services'):
            serviceName = name
        elif granularity in ('Resource', 'Resources'):
            resourceName = name
        elif granularity in ('StorageElement', 'StorageElements'):
            storageElementName = name

        res = self.rsDB.getMonitoredsList(
            granularity,
            siteName=siteName,
            serviceName=serviceName,
            resourceName=resourceName,
            storageElementName=storageElementName)

        if res == []:
            return False
        else:
            return True
Esempio n. 4
0
class Synchronizer:

    #############################################################################

    def __init__(self, rsDBin=None, rmDBin=None):

        self.rsDB = rsDBin
        self.rmDB = rmDBin

        if self.rsDB == None and self.rmDB == None:
            from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
            from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
            self.rsDB = ResourceStatusDB()
            self.rmDB = ResourceManagementDB()

        self.GOCDBClient = GOCDBClient()

#############################################################################

#  def sync(self, thingsToSync = None, fake_param = None):

    def sync(self, _a, _b):
        """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """

        thingsToSync = [
            'Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements',
            'RegistryUsers'
        ]

        gLogger.info("!!! Sync DB content with CS content for %s !!!" %
                     (' '.join(x for x in thingsToSync)))

        for thing in thingsToSync:
            getattr(self, '_sync' + thing)()

        return S_OK()

#############################################################################

    def _syncUtils(self):
        """
    Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
    """

        statusIn = self.rsDB.getStatusList()
        #delete status not more in Utils
        for stIn in statusIn:
            if stIn not in ValidStatus:
                self.rsDB.removeStatus(stIn)
        #Add new status
        for s in ValidStatus:
            if s not in statusIn:
                self.rsDB.addStatus(s)

        for g in ('Site', 'Service', 'Resource'):
            typeIn = self.rsDB.getTypesList(g)
            if g == 'Site':
                typesList = ValidSiteType
            elif g == 'Service':
                typesList = ValidServiceType
            if g == 'Resource':
                typesList = ValidResourceType
            #delete types not more in Utils
            for tIn in typeIn:
                if tIn not in typesList:
                    self.rsDB.removeType(g, tIn)
            #Add new types
            for t in typesList:
                if t not in typeIn:
                    self.rsDB.addType(g, t)

#############################################################################

    def _syncSites(self):
        """
    Sync DB content with sites that are in the CS
    """

        # sites in the DB now
        sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName'])
        sitesIn = [s[0] for s in sitesIn]

        # sites in CS now
        sitesList = getSites()['Value']

        try:
            sitesList.remove('LCG.Dummy.ch')
        except ValueError:
            pass

        # remove sites from the DB not more in the CS
        for site in sitesIn:
            if site not in sitesList:
                self.rsDB.removeSite(site)

        # add to DB what is in CS now and wasn't before
        for site in sitesList:
            if site not in sitesIn:
                # DIRAC Tier
                tier = getSiteTier(site)['Value'][0]
                if tier == 0 or tier == '0':
                    t = 'T0'
                elif tier == 1 or tier == '1':
                    t = 'T1'
                elif tier == 3 or tier == '3':
                    t = 'T3'
                else:
                    t = 'T2'

                #Grid Name of the site
                gridSiteName = getGOCSiteName(site)
                if not gridSiteName['OK']:
                    raise RSSException, gridSiteName['Message']
                gridSiteName = gridSiteName['Value']

                #Grid Tier (with a workaround!)
                DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
                if not DIRACSitesOfGridSites['OK']:
                    raise RSSException, DIRACSitesOfGridSites['Message']
                DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
                if len(DIRACSitesOfGridSites) == 1:
                    gt = t
                else:
                    gt = self.__getGOCTier(DIRACSitesOfGridSites)

                self.rsDB.addOrModifySite(
                    site, t, gridSiteName, 'Active', 'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))

                self.rsDB.addOrModifyGridSite(gridSiteName, gt)

                sitesIn.append(site)

#############################################################################

    def _syncVOBOX(self):
        """
    Sync DB content with VOBoxes
    """

        # services in the DB now
        servicesIn = self.rsDB.getMonitoredsList('Service',
                                                 paramsList=['ServiceName'])
        servicesIn = [s[0] for s in servicesIn]

        for site in [
                'LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk',
                'LCG.GRIDKA.de', 'LCG.NIKHEF.nl'
        ]:
            service = 'VO-BOX@' + site
            if service not in servicesIn:
                self.rsDB.addOrModifyService(
                    service, 'VO-BOX', site, 'Active', 'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))

#############################################################################

    def _syncResources(self):

        # resources in the DB now
        resourcesIn = self.rsDB.getMonitoredsList('Resource',
                                                  paramsList=['ResourceName'])
        resourcesIn = [r[0] for r in resourcesIn]

        # services in the DB now
        servicesIn = self.rsDB.getMonitoredsList('Service',
                                                 paramsList=['ServiceName'])
        servicesIn = [s[0] for s in servicesIn]

        # Site-CE mapping in CS now
        siteCE = getSiteCEMapping('LCG')['Value']
        # Site-SE mapping in CS now
        siteSE = getSiteSEMapping('LCG')['Value']

        # CEs in CS now
        CEList = []
        for i in siteCE.values():
            for ce in i:
                if ce is None:
                    continue
                CEList.append(ce)

        # SEs in CS now
        SEList = []
        for i in siteSE.values():
            for x in i:
                SEList.append(x)

        # SE Nodes in CS now
        SENodeList = []
        for SE in SEList:
            node = getSENodes(SE)['Value'][0]
            if node is None:
                continue
            if node not in SENodeList:
                SENodeList.append(node)

        # LFC Nodes in CS now
        LFCNodeList_L = []
        LFCNodeList_C = []
        for site in getLFCSites()['Value']:
            for readable in ('ReadOnly', 'ReadWrite'):
                LFCNode = getLFCNode(site, readable)['Value']
                if LFCNode is None or LFCNode == []:
                    continue
                LFCNode = LFCNode[0]
                if readable == 'ReadWrite':
                    if LFCNode not in LFCNodeList_C:
                        LFCNodeList_C.append(LFCNode)
                elif readable == 'ReadOnly':
                    if LFCNode not in LFCNodeList_L:
                        LFCNodeList_L.append(LFCNode)

        # FTS Nodes in CS now
        FTSNodeList = []
        sitesWithFTS = getFTSSites()
        for site in sitesWithFTS['Value']:
            fts = getFTSEndpoint(site)['Value']
            if fts is None or fts == []:
                continue
            fts = fts[0]
            if fts not in FTSNodeList:
                FTSNodeList.append(fts)

        # VOMS Nodes in CS now
        VOMSNodeList = getVOMSEndpoints()['Value']

        # complete list of resources in CS now
        resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList

        # list of services in CS now (to be done)
        servicesList = []

        #remove resources no more in the CS
        for res in resourcesIn:
            if res not in resourcesList:
                self.rsDB.removeResource(res)
                self.rsDB.removeStorageElement(resourceName=res)

        # add to DB what is in CS now and wasn't before

        # CEs
        for site in siteCE.keys():
            if site == 'LCG.Dummy.ch':
                continue
            for ce in siteCE[site]:
                if ce is None:
                    continue
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', ce)
                if not siteInGOCDB['OK']:
                    raise RSSException, siteInGOCDB['Message']
                if siteInGOCDB['Value'] == []:
                    trueName = socket.gethostbyname_ex(ce)[0]
                    siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                        'hostname', trueName)
                try:
                    siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
                except IndexError:
                    continue
                serviceType = 'Computing'
                service = serviceType + '@' + site

                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)

                if ce not in resourcesIn:
                    CEType = getCEType(site, ce)['Value']
                    ceType = 'CE'
                    if CEType == 'CREAM':
                        ceType = 'CREAMCE'
                    self.rsDB.addOrModifyResource(
                        ce, ceType, serviceType, site, siteInGOCDB, 'Active',
                        'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    resourcesIn.append(ce)

        # SRMs
        for srm in SENodeList:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', srm)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(srm)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)

            if srm not in resourcesIn and srm is not None:
                self.rsDB.addOrModifyResource(
                    srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(srm)

        # LFC_C
        for lfc in LFCNodeList_C:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', lfc)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(lfc)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if lfc not in resourcesIn and lfc is not None:
                self.rsDB.addOrModifyResource(
                    lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(lfc)

        # LFC_L
        for lfc in LFCNodeList_L:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', lfc)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(lfc)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if lfc not in resourcesIn and lfc is not None:
                self.rsDB.addOrModifyResource(
                    lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(lfc)

        # FTSs
        for fts in FTSNodeList:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', fts)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(fts)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            sites = siteInDIRAC['Value']
            serviceType = 'Storage'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if fts not in resourcesIn and fts is not None:
                self.rsDB.addOrModifyResource(
                    fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(fts)

        # VOMSs
        for voms in VOMSNodeList:
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', voms)
            if not siteInGOCDB['OK']:
                raise RSSException, siteInGOCDB['Message']
            if siteInGOCDB['Value'] == []:
                trueName = socket.gethostbyname_ex(voms)[0]
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', trueName)
            try:
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
            except IndexError:
                continue
            siteInDIRAC = getDIRACSiteName(siteInGOCDB)
            if not siteInDIRAC['OK']:
                raise RSSException, siteInDIRAC['Message']
            site = siteInDIRAC['Value']
            serviceType = 'VOMS'
            for site in sites:
                service = serviceType + '@' + site
                if service not in servicesList:
                    servicesList.append(service)
                if service not in servicesIn:
                    self.rsDB.addOrModifyService(
                        service, serviceType, site, 'Active', 'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                    servicesIn.append(service)
            if voms not in resourcesIn and voms is not None:
                self.rsDB.addOrModifyResource(
                    voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active',
                    'init',
                    datetime.datetime.utcnow().replace(microsecond=0),
                    'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
                resourcesIn.append(voms)

        #remove services no more in the CS
        for ser in servicesIn:
            if ser not in servicesList:
                serType = ser.split('@')[0]
                if serType != 'VO-BOX':
                    self.rsDB.removeService(ser)
                    self.rsDB.removeResource(serviceName=ser)
                    site = ser.split('@')[1]
                    if serType == 'Storage':
                        self.rsDB.removeStorageElement(siteName=site)

#############################################################################

    def _syncStorageElements(self):

        # Get StorageElements from the CS
        SEs = getStorageElements()
        if not SEs['OK']:
            raise RSSException, SEs['Message']
        SEs = SEs['Value']

        for access in ('Read', 'Write'):

            storageElementsIn = self.rsDB.getMonitoredsList(
                'StorageElement' + access, paramsList=['StorageElementName'])
            try:
                storageElementsIn = [x[0] for x in storageElementsIn]
            except IndexError:
                pass

            #remove storageElements no more in the CS
            for se in storageElementsIn:
                if se not in SEs:
                    self.rsDB.removeStorageElement(storageElementName=se,
                                                   resourceName=None,
                                                   access=access)

            #Add new storage Elements
            for SE in SEs:
                srm = getSENodes(SE)['Value'][0]
                if srm == None:
                    continue
                siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                    'hostname', srm)
                if not siteInGOCDB['OK']:
                    raise RSSException, siteInGOCDB['Message']
                if siteInGOCDB['Value'] == []:
                    continue
                siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']

                if SE not in storageElementsIn:
                    self.rsDB.addOrModifyStorageElement(
                        SE,
                        srm,
                        siteInGOCDB,
                        'Active',
                        'init',
                        datetime.datetime.utcnow().replace(microsecond=0),
                        'RS_SVC',
                        datetime.datetime(9999, 12, 31, 23, 59, 59),
                        access=access)
                    storageElementsIn.append(SE)

#############################################################################

    def __getGOCTier(self, sitesList):

        gridTier = 3

        for site in sitesList:

            tier = getSiteTier(site)['Value'][0]

            if tier == 0 or tier == '0':
                tn = 0
            elif tier == 1 or tier == '1':
                tn = 1
            elif tier == 3 or tier == '3':
                tn = 3
            else:
                tn = 2

            if tn < gridTier:
                gridTier = tn

        if gridTier == 0:
            gt = 'T0'
        elif gridTier == 1:
            gt = 'T1'
        elif gridTier == 3:
            gt = 'T3'
        else:
            gt = 'T2'

        return gt


#############################################################################

    def _syncRegistryUsers(self):
        from DIRAC.ResourceStatusSystem.Utilities import CS
        users = CS.getTypedDictRootedAt("Users", root="/Registry")
        for u in users:
            if type(users[u]['DN']) == list:
                users[u]['DN'] = users[u]['DN'][0]
            if type(users[u]['Email']) == list:
                users[u]['Email'] = users[u]['Email'][0]

            users[u]['DN'] = users[u]['DN'].split('=')[-1]
            self.rmDB.registryAddUser(u, users[u]['DN'].lower(),
                                      users[u]['Email'].lower())