예제 #1
0
class Synchronizer(object):
    def __init__(self, rsClient=None, rmClient=None):

        self.GOCDBClient = GOCDBClient()
        self.rsClient = ResourceStatusClient(
        ) if rsClient == None else rsClient
        self.rmClient = ResourceManagementClient(
        ) if rmClient == None else rmClient

        self.synclist = [
            'Sites', 'Resources', 'StorageElements', 'Services',
            'RegistryUsers'
        ]

################################################################################

    def sync(self, _a, _b):
        """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """
        gLogger.info("!!! Sync DB content with CS content for %s !!!" %
                     (", ".join(self.synclist)))

        for thing in self.synclist:
            getattr(self, '_sync' + thing)()

        return S_OK()

################################################################################

    def __purge_resource(self, resourceName):
        # Maybe remove attached SEs

        #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName))
        SEs = self.rsClient.getStorageElement(resourceName=resourceName)
        if not SEs['OK']:
            gLogger.error(SEs['Message'])
            return SEs

        #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs]))
        SEs = [se[0] for se in SEs]
        res = self.rsClient.removeElement('StorageElement', SEs)
        if not res['OK']:
            gLogger.error(res['Message'])
            return res

        # Remove resource itself.
        #Utils.unpack(self.rsClient.removeElement("Resource", resourceName))
        res = self.rsClient.removeElement('Resource', resourceName)
        if not res['OK']:
            gLogger.error(res['Message'])

        return res

    def __purge_site(self, siteName):
        # Remove associated resources and services

        #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName))
        resources = self.rsClient.getResource(siteName=siteName)
        if not resources['OK']:
            gLogger.error(resources['Message'])
            return resources

        #services  = Utils.unpack(self.rsClient.getService(siteName=siteName))
        services = self.rsClient.getService(siteName=siteName)
        if not services['OK']:
            gLogger.error(services['Message'])
            return services

        #_ = [self.__purge_resource(r[0]) for r in resources]
        for resource in resources:
            res = self.__purge_resource(resource[0])
            if not res['OK']:
                gLogger.error(res['Message'])
                return res

        #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services]))
        services = [service[0] for service in services['Value']]
        res = self.rsClient.removeElement('Service', services)
        if not res['OK']:
            gLogger.error(res['Message'])
            return res

        # Remove site itself
        #Utils.unpack(self.rsClient.removeElement("Site", siteName))
        res = self.rsClient.removeElement('Site', siteName)
        if not res['OK']:
            gLogger.info(res['Message'])

        return res

    def _syncSites(self):
        """
    Sync DB content with sites that are in the CS
    """
        def getGOCTier(sitesList):
            return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))

        # sites in the DB now
        #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))

        sites = self.rsClient.getSite()
        if not sites['OK']:
            gLogger.error(sites['Message'])
            return sites
        sitesDB = set([site[0] for site in sites['Value']])

        # sites in CS now
        sitesCS = set(CS.getSites())

        gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" %
                     (len(sitesCS), len(sitesDB)))

        # remove sites and associated resources, services, and storage
        # elements from the DB that are not in the CS:
        for s in sitesDB - sitesCS:
            gLogger.info("Purging Site %s (not in CS anymore)" % s)
            self.__purge_site(s)

        # add to DB what is missing
        gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
        for site in sitesCS - sitesDB:
            siteType = site.split(".")[0]
            # DIRAC Tier
            tier = "T" + str(CS.getSiteTier(site))
            if siteType == "LCG":
                # Grid Name of the site
                #gridSiteName = Utils.unpack(getGOCSiteName(site))
                gridSiteName = getGOCSiteName(site)
                if not gridSiteName['OK']:
                    gLogger.error(gridSiteName['Message'])
                    return gridSiteName
                gridSiteName = gridSiteName['Value']

                # Grid Tier (with a workaround!)
                #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
                DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
                if not DIRACSitesOfGridSites['OK']:
                    gLogger.error(DIRACSitesOfGridSites['Message'])
                    return DIRACSitesOfGridSites
                DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']

                if len(DIRACSitesOfGridSites) == 1:
                    gt = tier
                else:
                    gt = getGOCTier(DIRACSitesOfGridSites)

                #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
                res = self.rsClient.addOrModifyGridSite(gridSiteName, gt)
                if not res['OK']:
                    gLogger.error(res['Message'])
                    return res

                #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
                res = self.rsClient.addOrModifySite(site, tier, gridSiteName)
                if not res['OK']:
                    gLogger.error(res['Message'])
                    return res

            elif siteType == "DIRAC":
                #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
                res = self.rsClient.addOrModifySite(site, tier, "NULL")
                if not res['OK']:
                    gLogger.error(res['Message'])
                    return res

################################################################################
# _syncResources HELPER functions

    def __updateService(self, site, type_):
        service = type_ + '@' + site
        #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site )
        res = self.rsClient.addOrModifyService(service, type_, site)
        if not res['OK']:
            gLogger.error(res['Message'])
            return res

    def __getServiceEndpointInfo(self, node):
        #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) )
        res = self.GOCDBClient.getServiceEndpointInfo('hostname', node)
        if res['OK']:
            res = res['Value']
        else:
            gLogger.warn('Error getting hostname info for %s' % node)
            return []

        if res == []:
            #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) )
            url = Utils.canonicalURL(node)
            res = self.GOCDBClient.getServiceEndpointInfo('hostname', url)
            if res['OK']:
                res = res['Value']
            else:
                gLogger.warn('Error getting canonical hostname info for %s' %
                             node)
                res = []

        return res

    def __syncNode(self,
                   NodeInCS,
                   resourcesInDB,
                   resourceType,
                   serviceType,
                   site="NULL"):

        nodesToUpdate = NodeInCS - resourcesInDB
        if len(nodesToUpdate) > 0:
            gLogger.debug(str(NodeInCS))
            gLogger.debug(str(nodesToUpdate))

        # Update Service table
        siteInGOCDB = [
            self.__getServiceEndpointInfo(node) for node in nodesToUpdate
        ]
        siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
        #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]

        sites = []
        for sInGOCDB in siteInGOCDB:
            siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME'])
            if not siteName['OK']:
                gLogger.error(siteName['Message'])
                return siteName
            sites.append(siteName['Value'])

        sites = Utils.list_sanitize(Utils.list_flatten(sites))
        _ = [self.__updateService(s, serviceType) for s in sites]

        # Update Resource table
        for node in NodeInCS:
            if serviceType == "Computing":
                resourceType = CS.getCEType(site, node)
            if node not in resourcesInDB and node is not None:
                try:
                    siteInGOCDB = self.__getServiceEndpointInfo(
                        node)[0]['SITENAME']
                except IndexError:  # No INFO in GOCDB: Node does not exist
                    gLogger.warn(
                        "Node %s is not in GOCDB!! Considering that it does not exists!"
                        % node)
                    continue

                assert (type(siteInGOCDB) == str)
                #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
                res = self.rsClient.addOrModifyResource(
                    node, resourceType, serviceType, site, siteInGOCDB)
                if not res['OK']:
                    gLogger.error(res['Message'])
                    return res
                resourcesInDB.add(node)

################################################################################

    def _syncResources(self):
        gLogger.info("Starting sync of Resources")

        # resources in the DB now
        #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource())))

        resources = self.rsClient.getResource()
        if not resources['OK']:
            gLogger.error(resources['Message'])
            return resources

        resourcesInDB = set([resource[0] for resource in resources['Value']])

        # Site-CE / Site-SE mapping in CS now
        #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' ))
        CEinCS = getSiteCEMapping('LCG')
        if not CEinCS['OK']:
            gLogger.error(CEinCS['Message'])
            return CEinCS
        CEinCS = CEinCS['Value']

        # All CEs in CS now
        CEInCS = Utils.set_sanitize(
            [CE for celist in CEinCS.values() for CE in celist])

        # All SE Nodes in CS now
        SENodeInCS = set(CS.getSENodes())

        # LFC Nodes in CS now
        LFCNodeInCS_L = set(CS.getLFCNode(readable="ReadOnly"))
        LFCNodeInCS_C = set(CS.getLFCNode(readable="ReadWrite"))

        # FTS Nodes in CS now
        FTSNodeInCS = set([
            v.split("/")[2][0:-5] for v in CS.getTypedDictRootedAt(
                root="/Resources/FTSEndpoints").values()
        ])

        # VOMS Nodes in CS now
        VOMSNodeInCS = set(CS.getVOMSEndpoints())

        # complete list of resources in CS now
        resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS

        gLogger.info(
            "  %d resources in CS, %s resources in DB, updating %d resources" %
            (len(resourcesInCS), len(resourcesInDB),
             len(resourcesInCS) - len(resourcesInDB)))

        # Remove resources that are not in the CS anymore
        for res in resourcesInDB - resourcesInCS:
            gLogger.info("Purging resource %s. Reason: not in CS anywore." %
                         res)
            self.__purge_resource(res)

        # Add to DB what is in CS now and wasn't before

        # CEs
        for site in CEinCS:
            self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing",
                            site)

        # SRMs
        self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage")

        # LFC_C
        self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage")

        # LFC_L
        self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage")

        # FTSs
        self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage")

        # VOMSs
        self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS")

################################################################################

    def _syncStorageElements(self):

        # Get StorageElements from the CS and the DB
        CSSEs = set(CS.getSEs())
        #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement())))
        ses = self.rsClient.getStorageElement()
        if not ses['OK']:
            gLogger.error(ses['Message'])
            return ses

        DBSEs = set([se[0] for se in ses['Value']])

        # Remove storageElements that are in DB but not in CS
        for se in DBSEs - CSSEs:
            #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se )
            res = self.rsClient.removeElement('StorageElement', se)
            if not res['OK']:
                gLogger.error(res['Message'])
                return res

        # Add new storage elements
        gLogger.info(
            "Updating %d StorageElements in DB (%d on CS vs %d on DB)" %
            (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs)))
        for SE in CSSEs - DBSEs:
            srm = CS.getSEHost(SE)
            if not srm:
                gLogger.warn("%s has no srm URL in CS!!!" % SE)
                continue
            #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ))
            siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
                'hostname', srm)
            if siteInGOCDB['OK']:
                siteInGOCDB = siteInGOCDB['Value']
            else:
                gLogger.error("Error getting hostname for %s from GOCDB!!!" %
                              srm)
                continue
            if siteInGOCDB == []:
                gLogger.warn("%s is not in GOCDB!!!" % srm)
                continue
            siteInGOCDB = siteInGOCDB[0]['SITENAME']
            #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB )
            res = self.rsClient.addOrModifyStorageElement(SE, srm, siteInGOCDB)
            if not res['OK']:
                gLogger.error(res['Message'])
                return res

################################################################################

    def _syncServices(self):
        """This function is in charge of cleaning the Service table in DB
    in case of obsolescence."""
        # services in the DB now
        #servicesInDB = Utils.unpack(self.rsClient.getService())
        servicesInDB = self.rsClient.getService()
        if not servicesInDB['OK']:
            gLogger.error(servicesInDB['Message'])
            return servicesInDB
        servicesInDB = servicesInDB['Value']

        for service_name, service_type, site_name in servicesInDB:
            if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]:

                #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []:
                resource = self.rsClient.getResource(siteName=site_name,
                                                     serviceType=service_type)
                if not resource['OK']:
                    gLogger.error(resource['Message'])
                    return resource
                if resource['Value'] == []:

                    gLogger.info(
                        "Deleting Service %s since it has no corresponding resources."
                        % service_name)
                    #Utils.protect2(self.rsClient.removeElement, "Service", service_name)
                    res = self.rsClient.removeElement("Service", service_name)
                    if not res['OK']:
                        gLogger.error(res['Message'])
                        return res
            elif service_type == "Storage":
                res = self.rsClient.getSite(siteName=site_name,
                                            meta={'columns': 'GridSiteName'})
                if res['OK']:
                    res = res['Value']
                else:
                    res = []

                if res:
                    if self.rsClient.getResource(
                            gridSiteName=res[0],
                            serviceType=service_type) == []:
                        gLogger.info(
                            "Deleting Service %s since it has no corresponding resources."
                            % service_name)
                        #Utils.protect2(self.rsClient.removeElement, "Service", service_name)
                        res = self.rsClient.removeElement(
                            "Service", service_name)
                        if not res['OK']:
                            gLogger.error(res['Message'])
                            return res

    def _syncRegistryUsers(self):
        users = CS.getTypedDictRootedAt("Users", root="/Registry")
        usersInCS = set(users.keys())
        #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache())))

        usersInCache = self.rmClient.getUserRegistryCache()
        if not usersInCache['OK']:
            gLogger.error(usersInCache['Message'])
            return usersInCache

        usersInDB = set(
            [userInCache[0] for userInCache in usersInCache['Value']])

        usersToAdd = usersInCS - usersInDB
        usersToDel = usersInDB - usersInCS

        gLogger.info("Updating Registry Users: + %d, - %d" %
                     (len(usersToAdd), len(usersToDel)))
        if len(usersToAdd) > 0:
            gLogger.debug(str(usersToAdd))
        if len(usersToDel) > 0:
            gLogger.debug(str(usersToDel))

        for u in usersToAdd:
            if type(users[u]['DN']) == list:
                users[u]['DN'] = users[u]['DN'][0]
            if type(users[u]['Email']) == list:
                users[u]['Email'] = users[u]['Email'][0]
            users[u]['DN'] = users[u]['DN'].split('=')[-1]

            #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower()))

            res = self.rmClient.addOrModifyUserRegistryCache(
                u, users[u]['DN'], users[u]['Email'].lower())
            if not res['OK']:
                gLogger.error(res['Message'])
                return res

        for u in usersToDel:
            #Utils.protect2(self.rmClient.deleteUserRegistryCache, u)
            res = self.rmClient.deleteUserRegistryCache(u)
            if not res['OK']:
                gLogger.error(res['Message'])
                return res


################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
예제 #2
0
파일: Synchronizer.py 프로젝트: bmb/DIRAC
class Synchronizer( object ):

  def __init__( self, rsClient = None, rmClient = None ):

    self.GOCDBClient = GOCDBClient()
    self.rsClient = ResourceStatusClient()     if rsClient == None else rsClient
    self.rmClient = ResourceManagementClient() if rmClient == None else rmClient

    self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ]

################################################################################

  def sync( self, _a, _b ):
    """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """
    gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ", ".join(self.synclist) ) )

    for thing in self.synclist:
      getattr( self, '_sync' + thing )()

    return S_OK()

################################################################################
  def __purge_resource( self, resourceName ):
    # Maybe remove attached SEs
    
    #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName))
    SEs = self.rsClient.getStorageElement( resourceName = resourceName )
    if not SEs[ 'OK' ]:
      gLogger.error( SEs[ 'Message' ] )
      return SEs
    
    #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs]))   
    SEs = [ se[0] for se in SEs ]  
    res = self.rsClient.removeElement( 'StorageElement', SEs )
    if not res[ 'OK' ]:
      gLogger.error( res[ 'Message' ] )
      return res
    
    # Remove resource itself.
    #Utils.unpack(self.rsClient.removeElement("Resource", resourceName))
    res = self.rsClient.removeElement( 'Resource', resourceName )
    if not res[ 'OK' ]:
      gLogger.error( res[ 'Message' ] ) 
    
    return res
    
  def __purge_site( self, siteName ):
    # Remove associated resources and services
    
    #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName))
    resources = self.rsClient.getResource( siteName = siteName )
    if not resources[ 'OK' ]:
      gLogger.error( resources[ 'Message' ] )
      return resources
    
    #services  = Utils.unpack(self.rsClient.getService(siteName=siteName))
    services = self.rsClient.getService( siteName = siteName )
    if not services[ 'OK' ]:
      gLogger.error( services[ 'Message' ] )
      return services
    
    #_ = [self.__purge_resource(r[0]) for r in resources]
    for resource in resources:
      res = self.__purge_resource( resource[ 0 ] )
      if not res[ 'OK' ]:
        gLogger.error( res[ 'Message' ] )
        return res
       
    #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services]))
    services = [ service[ 0 ] for service in services[ 'Value' ] ]
    res      = self.rsClient.removeElement( 'Service', services )
    if not res[ 'OK' ]:
      gLogger.error( res[ 'Message' ] )
      return res  
    
    # Remove site itself
    #Utils.unpack(self.rsClient.removeElement("Site", siteName))
    res = self.rsClient.removeElement( 'Site', siteName )
    if not res[ 'OK' ]:
      gLogger.info( res[ 'Message' ] )
    
    return res
    
  def _syncSites( self ):
    """
    Sync DB content with sites that are in the CS
    """
    def getGOCTier(sitesList):
      return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))

    # sites in the DB now
    #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))
    
    sites = self.rsClient.getSite()
    if not sites[ 'OK' ]:
      gLogger.error( sites[ 'Message' ] )
      return sites
    sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] )

    # sites in CS now
    sitesCS = set( CS.getSites() )

    gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB)))

    # remove sites and associated resources, services, and storage
    # elements from the DB that are not in the CS:
    for s in sitesDB - sitesCS:
      gLogger.info("Purging Site %s (not in CS anymore)" % s)
      self.__purge_site(s)

    # add to DB what is missing
    gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
    for site in sitesCS - sitesDB:
      siteType = site.split(".")[0]
      # DIRAC Tier
      tier = "T" + str(CS.getSiteTier( site ))
      if siteType == "LCG":
        # Grid Name of the site
        #gridSiteName = Utils.unpack(getGOCSiteName(site))
        gridSiteName = getGOCSiteName( site )
        if not gridSiteName[ 'OK' ]:
          gLogger.error( gridSiteName[ 'Message' ] )
          return gridSiteName
        gridSiteName = gridSiteName[ 'Value' ]

        # Grid Tier (with a workaround!)
        #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
        DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
        if not DIRACSitesOfGridSites[ 'OK' ]:
          gLogger.error( DIRACSitesOfGridSites[ 'Message' ] )
          return DIRACSitesOfGridSites
        DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ]
        
        if len( DIRACSitesOfGridSites ) == 1:
          gt = tier
        else:
          gt = getGOCTier( DIRACSitesOfGridSites )

        #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
        res = self.rsClient.addOrModifyGridSite( gridSiteName, gt )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res
        
        #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
        res = self.rsClient.addOrModifySite( site, tier, gridSiteName )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res

      elif siteType == "DIRAC":
        #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
        res = self.rsClient.addOrModifySite( site, tier, "NULL" )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res

################################################################################
# _syncResources HELPER functions

  def __updateService(self, site, type_):
    service = type_ + '@' + site
    #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site )
    res = self.rsClient.addOrModifyService( service, type_, site )
    if not res[ 'OK' ]:
      gLogger.error( res[ 'Message' ] )
      return res

  def __getServiceEndpointInfo(self, node):
    #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) )
    res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node )
    if res['OK']:
      res = res[ 'Value' ]
    else:
      gLogger.warn( 'Error getting hostname info for %s' % node )
      return []
        
    if res == []:
      #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) )
      url = Utils.canonicalURL(node)
      res = self.GOCDBClient.getServiceEndpointInfo('hostname', url )
      if res['OK']:
        res = res[ 'Value' ]
      else:
        gLogger.warn( 'Error getting canonical hostname info for %s' % node )
        res = []
      
    return res

  def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"):

    nodesToUpdate = NodeInCS - resourcesInDB
    if len(nodesToUpdate) > 0:
      gLogger.debug(str(NodeInCS))
      gLogger.debug(str(nodesToUpdate))

    # Update Service table
    siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate]
    siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
    #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
    
    sites = []
    for sInGOCDB in siteInGOCDB:
      siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] )
      if not siteName[ 'OK' ]:
        gLogger.error( siteName[ 'Message' ] )
        return siteName
      sites.append( siteName[ 'Value' ] )
    
    sites = Utils.list_sanitize( Utils.list_flatten( sites ) )
    _     = [ self.__updateService(s, serviceType) for s in sites ]

    # Update Resource table
    for node in NodeInCS:
      if serviceType == "Computing":
        resourceType = CS.getCEType(site, node)
      if node not in resourcesInDB and node is not None:
        try:
          siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME']
        except IndexError: # No INFO in GOCDB: Node does not exist
          gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node)
          continue

        assert(type(siteInGOCDB) == str)
        #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
        res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res
        resourcesInDB.add( node )

################################################################################

  def _syncResources( self ):
    gLogger.info("Starting sync of Resources")

    # resources in the DB now
    #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource())))
    
    resources = self.rsClient.getResource()
    if not resources[ 'OK' ]:
      gLogger.error( resources[ 'Message' ] )
      return resources
    
    resourcesInDB = set( [ resource[ 0 ] for resource in resources[ 'Value' ] ] )
      
    # Site-CE / Site-SE mapping in CS now
    #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' ))
    CEinCS = getSiteCEMapping( 'LCG' )
    if not CEinCS[ 'OK' ]:
      gLogger.error( CEinCS[ 'Message' ] )
      return CEinCS
    CEinCS = CEinCS[ 'Value' ]

    # All CEs in CS now
    CEInCS = Utils.set_sanitize([CE for celist in CEinCS.values() for CE in celist])

    # All SE Nodes in CS now
    SENodeInCS = set(CS.getSENodes())

    # LFC Nodes in CS now
    LFCNodeInCS_L = set(CS.getLFCNode(readable = "ReadOnly"))
    LFCNodeInCS_C = set(CS.getLFCNode(readable = "ReadWrite"))

    # FTS Nodes in CS now
    FTSNodeInCS = set([v.split("/")[2][0:-5] for v
                       in CS.getTypedDictRootedAt(root="/Resources/FTSEndpoints").values()])

    # VOMS Nodes in CS now
    VOMSNodeInCS = set(CS.getVOMSEndpoints())

    # complete list of resources in CS now
    resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS

    gLogger.info("  %d resources in CS, %s resources in DB, updating %d resources" %
                 (len(resourcesInCS), len(resourcesInDB), len(resourcesInCS)-len(resourcesInDB)))

    # Remove resources that are not in the CS anymore
    for res in resourcesInDB - resourcesInCS:
      gLogger.info("Purging resource %s. Reason: not in CS anywore." % res)
      self.__purge_resource(res)

    # Add to DB what is in CS now and wasn't before

    # CEs
    for site in CEinCS:
      self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site)

    # SRMs
    self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage")

    # LFC_C
    self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage")

    # LFC_L
    self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage")

    # FTSs
    self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage")

    # VOMSs
    self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS")

################################################################################

  def _syncStorageElements( self ):

    # Get StorageElements from the CS and the DB
    CSSEs = set(CS.getSEs())
    #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement())))
    ses = self.rsClient.getStorageElement()
    if not ses[ 'OK' ]:
      gLogger.error( ses[ 'Message' ] )
      return ses
    
    DBSEs = set( [ se[0] for se in ses[ 'Value' ] ] )  

    # Remove storageElements that are in DB but not in CS
    for se in DBSEs - CSSEs:
      #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se )
      res = self.rsClient.removeElement( 'StorageElement', se )
      if not res[ 'OK' ]:
        gLogger.error( res[ 'Message' ] )
        return res

    # Add new storage elements
    gLogger.info("Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs)))
    for SE in CSSEs - DBSEs:
      srm = CS.getSEHost( SE )
      if not srm:
        gLogger.warn("%s has no srm URL in CS!!!" % SE)
        continue
      #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ))
      siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
      if siteInGOCDB[ 'OK' ]:
        siteInGOCDB = siteInGOCDB[ 'Value' ]
      else:
        gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm)
        continue
      if siteInGOCDB == []:
        gLogger.warn("%s is not in GOCDB!!!" % srm)
        continue
      siteInGOCDB = siteInGOCDB[ 0 ][ 'SITENAME' ]
      #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB )
      res = self.rsClient.addOrModifyStorageElement( SE, srm, siteInGOCDB )
      if not res[ 'OK' ]:
        gLogger.error( res[ 'Message' ] )
        return res

################################################################################

  def _syncServices(self):
    """This function is in charge of cleaning the Service table in DB
    in case of obsolescence."""
    # services in the DB now
    #servicesInDB = Utils.unpack(self.rsClient.getService())
    servicesInDB = self.rsClient.getService()
    if not servicesInDB[ 'OK' ]:
      gLogger.error( servicesInDB[ 'Message' ] )
      return servicesInDB
    servicesInDB = servicesInDB[ 'Value' ]
    
    for service_name, service_type, site_name in servicesInDB:
      if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]:
        
        #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []:
        resource = self.rsClient.getResource( siteName = site_name, serviceType = service_type )
        if not resource[ 'OK' ]:
          gLogger.error( resource[ 'Message' ] )
          return resource
        if resource[ 'Value' ] == []:
          
          gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name)
          #Utils.protect2(self.rsClient.removeElement, "Service", service_name)
          res = self.rsClient.removeElement( "Service", service_name )
          if not res[ 'OK' ]:
            gLogger.error( res[ 'Message' ] )
            return res
      elif service_type == "Storage":
        res = self.rsClient.getSite( siteName = site_name, meta = { 'columns' : 'GridSiteName'} )
        if res[ 'OK' ]:
          res = res[ 'Value' ]
        else:
          res = []
        
        if res:
          if self.rsClient.getResource( gridSiteName = res[0], serviceType = service_type ) == []:
            gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name)
            #Utils.protect2(self.rsClient.removeElement, "Service", service_name)
            res = self.rsClient.removeElement( "Service", service_name )
            if not res[ 'OK' ]:
              gLogger.error( res[ 'Message' ] )
              return res

  def _syncRegistryUsers(self):
    users = CS.getTypedDictRootedAt("Users", root= "/Registry")
    usersInCS = set(users.keys())
    #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache())))
    
    usersInCache = self.rmClient.getUserRegistryCache()
    if not usersInCache[ 'OK' ]:
      gLogger.error( usersInCache[ 'Message' ] )
      return usersInCache
    
    usersInDB = set( [ userInCache[ 0 ] for userInCache in usersInCache[ 'Value' ] ] )
    
    usersToAdd = usersInCS - usersInDB
    usersToDel = usersInDB - usersInCS

    gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel)))
    if len(usersToAdd) > 0:
      gLogger.debug(str(usersToAdd))
    if len(usersToDel) > 0:
      gLogger.debug(str(usersToDel))

    for u in usersToAdd:
      if type(users[u]['DN']) == list:
        users[u]['DN'] = users[u]['DN'][0]
      if type(users[u]['Email']) == list:
        users[u]['Email'] = users[u]['Email'][0]
      users[u]['DN'] = users[u]['DN'].split('=')[-1]
      
      #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower()))
      
      res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower() )
      if not res[ 'OK' ]:
        gLogger.error( res[ 'Message' ] )
        return res

    for u in usersToDel:
      #Utils.protect2(self.rmClient.deleteUserRegistryCache, u)
      res = self.rmClient.deleteUserRegistryCache( u )
      if not res[ 'OK' ]:
        gLogger.error( res[ 'Message' ] )
        return res

################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF