class Synchronizer(object): def __init__(self, rsClient=None, rmClient=None): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient( ) if rsClient == None else rsClient self.rmClient = ResourceManagementClient( ) if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ] ################################################################################ def sync(self, _a, _b): """ :params: :attr:`thingsToSync`: list of things to sync """ gLogger.info("!!! Sync DB content with CS content for %s !!!" % (", ".join(self.synclist))) for thing in self.synclist: getattr(self, '_sync' + thing)() return S_OK() ################################################################################ def __purge_resource(self, resourceName): # Maybe remove attached SEs #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName)) SEs = self.rsClient.getStorageElement(resourceName=resourceName) if not SEs['OK']: gLogger.error(SEs['Message']) return SEs #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs])) SEs = [se[0] for se in SEs] res = self.rsClient.removeElement('StorageElement', SEs) if not res['OK']: gLogger.error(res['Message']) return res # Remove resource itself. #Utils.unpack(self.rsClient.removeElement("Resource", resourceName)) res = self.rsClient.removeElement('Resource', resourceName) if not res['OK']: gLogger.error(res['Message']) return res def __purge_site(self, siteName): # Remove associated resources and services #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName)) resources = self.rsClient.getResource(siteName=siteName) if not resources['OK']: gLogger.error(resources['Message']) return resources #services = Utils.unpack(self.rsClient.getService(siteName=siteName)) services = self.rsClient.getService(siteName=siteName) if not services['OK']: gLogger.error(services['Message']) return services #_ = [self.__purge_resource(r[0]) for r in resources] for resource in resources: res = self.__purge_resource(resource[0]) if not res['OK']: gLogger.error(res['Message']) return res #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services])) services = [service[0] for service in services['Value']] res = self.rsClient.removeElement('Service', services) if not res['OK']: gLogger.error(res['Message']) return res # Remove site itself #Utils.unpack(self.rsClient.removeElement("Site", siteName)) res = self.rsClient.removeElement('Site', siteName) if not res['OK']: gLogger.info(res['Message']) return res def _syncSites(self): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites['OK']: gLogger.error(sites['Message']) return sites sitesDB = set([site[0] for site in sites['Value']]) # sites in CS now sitesCS = set(CS.getSites()) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier(site)) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName(site) if not gridSiteName['OK']: gLogger.error(gridSiteName['Message']) return gridSiteName gridSiteName = gridSiteName['Value'] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName) if not DIRACSitesOfGridSites['OK']: gLogger.error(DIRACSitesOfGridSites['Message']) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len(DIRACSitesOfGridSites) == 1: gt = tier else: gt = getGOCTier(DIRACSitesOfGridSites) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite(gridSiteName, gt) if not res['OK']: gLogger.error(res['Message']) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite(site, tier, gridSiteName) if not res['OK']: gLogger.error(res['Message']) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite(site, tier, "NULL") if not res['OK']: gLogger.error(res['Message']) return res ################################################################################ # _syncResources HELPER functions def __updateService(self, site, type_): service = type_ + '@' + site #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site ) res = self.rsClient.addOrModifyService(service, type_, site) if not res['OK']: gLogger.error(res['Message']) return res def __getServiceEndpointInfo(self, node): #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) ) res = self.GOCDBClient.getServiceEndpointInfo('hostname', node) if res['OK']: res = res['Value'] else: gLogger.warn('Error getting hostname info for %s' % node) return [] if res == []: #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) ) url = Utils.canonicalURL(node) res = self.GOCDBClient.getServiceEndpointInfo('hostname', url) if res['OK']: res = res['Value'] else: gLogger.warn('Error getting canonical hostname info for %s' % node) res = [] return res def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site="NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [ self.__getServiceEndpointInfo(node) for node in nodesToUpdate ] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME']) if not siteName['OK']: gLogger.error(siteName['Message']) return siteName sites.append(siteName['Value']) sites = Utils.list_sanitize(Utils.list_flatten(sites)) _ = [self.__updateService(s, serviceType) for s in sites] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo( node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn( "Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert (type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB) if not res['OK']: gLogger.error(res['Message']) return res resourcesInDB.add(node) ################################################################################ def _syncResources(self): gLogger.info("Starting sync of Resources") # resources in the DB now #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource()))) resources = self.rsClient.getResource() if not resources['OK']: gLogger.error(resources['Message']) return resources resourcesInDB = set([resource[0] for resource in resources['Value']]) # Site-CE / Site-SE mapping in CS now #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' )) CEinCS = getSiteCEMapping('LCG') if not CEinCS['OK']: gLogger.error(CEinCS['Message']) return CEinCS CEinCS = CEinCS['Value'] # All CEs in CS now CEInCS = Utils.set_sanitize( [CE for celist in CEinCS.values() for CE in celist]) # All SE Nodes in CS now SENodeInCS = set(CS.getSENodes()) # LFC Nodes in CS now LFCNodeInCS_L = set(CS.getLFCNode(readable="ReadOnly")) LFCNodeInCS_C = set(CS.getLFCNode(readable="ReadWrite")) # FTS Nodes in CS now FTSNodeInCS = set([ v.split("/")[2][0:-5] for v in CS.getTypedDictRootedAt( root="/Resources/FTSEndpoints").values() ]) # VOMS Nodes in CS now VOMSNodeInCS = set(CS.getVOMSEndpoints()) # complete list of resources in CS now resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS gLogger.info( " %d resources in CS, %s resources in DB, updating %d resources" % (len(resourcesInCS), len(resourcesInDB), len(resourcesInCS) - len(resourcesInDB))) # Remove resources that are not in the CS anymore for res in resourcesInDB - resourcesInCS: gLogger.info("Purging resource %s. Reason: not in CS anywore." % res) self.__purge_resource(res) # Add to DB what is in CS now and wasn't before # CEs for site in CEinCS: self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site) # SRMs self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage") # LFC_C self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage") # LFC_L self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage") # FTSs self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage") # VOMSs self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS") ################################################################################ def _syncStorageElements(self): # Get StorageElements from the CS and the DB CSSEs = set(CS.getSEs()) #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement()))) ses = self.rsClient.getStorageElement() if not ses['OK']: gLogger.error(ses['Message']) return ses DBSEs = set([se[0] for se in ses['Value']]) # Remove storageElements that are in DB but not in CS for se in DBSEs - CSSEs: #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se ) res = self.rsClient.removeElement('StorageElement', se) if not res['OK']: gLogger.error(res['Message']) return res # Add new storage elements gLogger.info( "Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs))) for SE in CSSEs - DBSEs: srm = CS.getSEHost(SE) if not srm: gLogger.warn("%s has no srm URL in CS!!!" % SE) continue #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )) siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if siteInGOCDB['OK']: siteInGOCDB = siteInGOCDB['Value'] else: gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm) continue if siteInGOCDB == []: gLogger.warn("%s is not in GOCDB!!!" % srm) continue siteInGOCDB = siteInGOCDB[0]['SITENAME'] #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB ) res = self.rsClient.addOrModifyStorageElement(SE, srm, siteInGOCDB) if not res['OK']: gLogger.error(res['Message']) return res ################################################################################ def _syncServices(self): """This function is in charge of cleaning the Service table in DB in case of obsolescence.""" # services in the DB now #servicesInDB = Utils.unpack(self.rsClient.getService()) servicesInDB = self.rsClient.getService() if not servicesInDB['OK']: gLogger.error(servicesInDB['Message']) return servicesInDB servicesInDB = servicesInDB['Value'] for service_name, service_type, site_name in servicesInDB: if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]: #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []: resource = self.rsClient.getResource(siteName=site_name, serviceType=service_type) if not resource['OK']: gLogger.error(resource['Message']) return resource if resource['Value'] == []: gLogger.info( "Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement("Service", service_name) if not res['OK']: gLogger.error(res['Message']) return res elif service_type == "Storage": res = self.rsClient.getSite(siteName=site_name, meta={'columns': 'GridSiteName'}) if res['OK']: res = res['Value'] else: res = [] if res: if self.rsClient.getResource( gridSiteName=res[0], serviceType=service_type) == []: gLogger.info( "Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name) if not res['OK']: gLogger.error(res['Message']) return res def _syncRegistryUsers(self): users = CS.getTypedDictRootedAt("Users", root="/Registry") usersInCS = set(users.keys()) #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache()))) usersInCache = self.rmClient.getUserRegistryCache() if not usersInCache['OK']: gLogger.error(usersInCache['Message']) return usersInCache usersInDB = set( [userInCache[0] for userInCache in usersInCache['Value']]) usersToAdd = usersInCS - usersInDB usersToDel = usersInDB - usersInCS gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel))) if len(usersToAdd) > 0: gLogger.debug(str(usersToAdd)) if len(usersToDel) > 0: gLogger.debug(str(usersToDel)) for u in usersToAdd: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower())) res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower()) if not res['OK']: gLogger.error(res['Message']) return res for u in usersToDel: #Utils.protect2(self.rmClient.deleteUserRegistryCache, u) res = self.rmClient.deleteUserRegistryCache(u) if not res['OK']: gLogger.error(res['Message']) return res ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class Synchronizer( object ): def __init__( self, rsClient = None, rmClient = None ): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient() if rsClient == None else rsClient self.rmClient = ResourceManagementClient() if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ] ################################################################################ def sync( self, _a, _b ): """ :params: :attr:`thingsToSync`: list of things to sync """ gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ", ".join(self.synclist) ) ) for thing in self.synclist: getattr( self, '_sync' + thing )() return S_OK() ################################################################################ def __purge_resource( self, resourceName ): # Maybe remove attached SEs #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName)) SEs = self.rsClient.getStorageElement( resourceName = resourceName ) if not SEs[ 'OK' ]: gLogger.error( SEs[ 'Message' ] ) return SEs #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs])) SEs = [ se[0] for se in SEs ] res = self.rsClient.removeElement( 'StorageElement', SEs ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Remove resource itself. #Utils.unpack(self.rsClient.removeElement("Resource", resourceName)) res = self.rsClient.removeElement( 'Resource', resourceName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def __purge_site( self, siteName ): # Remove associated resources and services #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName)) resources = self.rsClient.getResource( siteName = siteName ) if not resources[ 'OK' ]: gLogger.error( resources[ 'Message' ] ) return resources #services = Utils.unpack(self.rsClient.getService(siteName=siteName)) services = self.rsClient.getService( siteName = siteName ) if not services[ 'OK' ]: gLogger.error( services[ 'Message' ] ) return services #_ = [self.__purge_resource(r[0]) for r in resources] for resource in resources: res = self.__purge_resource( resource[ 0 ] ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services])) services = [ service[ 0 ] for service in services[ 'Value' ] ] res = self.rsClient.removeElement( 'Service', services ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Remove site itself #Utils.unpack(self.rsClient.removeElement("Site", siteName)) res = self.rsClient.removeElement( 'Site', siteName ) if not res[ 'OK' ]: gLogger.info( res[ 'Message' ] ) return res def _syncSites( self ): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites[ 'OK' ]: gLogger.error( sites[ 'Message' ] ) return sites sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] ) # sites in CS now sitesCS = set( CS.getSites() ) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier( site )) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName( site ) if not gridSiteName[ 'OK' ]: gLogger.error( gridSiteName[ 'Message' ] ) return gridSiteName gridSiteName = gridSiteName[ 'Value' ] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites[ 'OK' ]: gLogger.error( DIRACSitesOfGridSites[ 'Message' ] ) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ] if len( DIRACSitesOfGridSites ) == 1: gt = tier else: gt = getGOCTier( DIRACSitesOfGridSites ) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite( gridSiteName, gt ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite( site, tier, gridSiteName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite( site, tier, "NULL" ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ # _syncResources HELPER functions def __updateService(self, site, type_): service = type_ + '@' + site #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site ) res = self.rsClient.addOrModifyService( service, type_, site ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def __getServiceEndpointInfo(self, node): #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) ) res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) if res['OK']: res = res[ 'Value' ] else: gLogger.warn( 'Error getting hostname info for %s' % node ) return [] if res == []: #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) ) url = Utils.canonicalURL(node) res = self.GOCDBClient.getServiceEndpointInfo('hostname', url ) if res['OK']: res = res[ 'Value' ] else: gLogger.warn( 'Error getting canonical hostname info for %s' % node ) res = [] return res def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] ) if not siteName[ 'OK' ]: gLogger.error( siteName[ 'Message' ] ) return siteName sites.append( siteName[ 'Value' ] ) sites = Utils.list_sanitize( Utils.list_flatten( sites ) ) _ = [ self.__updateService(s, serviceType) for s in sites ] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert(type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res resourcesInDB.add( node ) ################################################################################ def _syncResources( self ): gLogger.info("Starting sync of Resources") # resources in the DB now #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource()))) resources = self.rsClient.getResource() if not resources[ 'OK' ]: gLogger.error( resources[ 'Message' ] ) return resources resourcesInDB = set( [ resource[ 0 ] for resource in resources[ 'Value' ] ] ) # Site-CE / Site-SE mapping in CS now #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' )) CEinCS = getSiteCEMapping( 'LCG' ) if not CEinCS[ 'OK' ]: gLogger.error( CEinCS[ 'Message' ] ) return CEinCS CEinCS = CEinCS[ 'Value' ] # All CEs in CS now CEInCS = Utils.set_sanitize([CE for celist in CEinCS.values() for CE in celist]) # All SE Nodes in CS now SENodeInCS = set(CS.getSENodes()) # LFC Nodes in CS now LFCNodeInCS_L = set(CS.getLFCNode(readable = "ReadOnly")) LFCNodeInCS_C = set(CS.getLFCNode(readable = "ReadWrite")) # FTS Nodes in CS now FTSNodeInCS = set([v.split("/")[2][0:-5] for v in CS.getTypedDictRootedAt(root="/Resources/FTSEndpoints").values()]) # VOMS Nodes in CS now VOMSNodeInCS = set(CS.getVOMSEndpoints()) # complete list of resources in CS now resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS gLogger.info(" %d resources in CS, %s resources in DB, updating %d resources" % (len(resourcesInCS), len(resourcesInDB), len(resourcesInCS)-len(resourcesInDB))) # Remove resources that are not in the CS anymore for res in resourcesInDB - resourcesInCS: gLogger.info("Purging resource %s. Reason: not in CS anywore." % res) self.__purge_resource(res) # Add to DB what is in CS now and wasn't before # CEs for site in CEinCS: self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site) # SRMs self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage") # LFC_C self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage") # LFC_L self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage") # FTSs self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage") # VOMSs self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS") ################################################################################ def _syncStorageElements( self ): # Get StorageElements from the CS and the DB CSSEs = set(CS.getSEs()) #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement()))) ses = self.rsClient.getStorageElement() if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) return ses DBSEs = set( [ se[0] for se in ses[ 'Value' ] ] ) # Remove storageElements that are in DB but not in CS for se in DBSEs - CSSEs: #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se ) res = self.rsClient.removeElement( 'StorageElement', se ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Add new storage elements gLogger.info("Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs))) for SE in CSSEs - DBSEs: srm = CS.getSEHost( SE ) if not srm: gLogger.warn("%s has no srm URL in CS!!!" % SE) continue #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )) siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if siteInGOCDB[ 'OK' ]: siteInGOCDB = siteInGOCDB[ 'Value' ] else: gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm) continue if siteInGOCDB == []: gLogger.warn("%s is not in GOCDB!!!" % srm) continue siteInGOCDB = siteInGOCDB[ 0 ][ 'SITENAME' ] #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB ) res = self.rsClient.addOrModifyStorageElement( SE, srm, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ def _syncServices(self): """This function is in charge of cleaning the Service table in DB in case of obsolescence.""" # services in the DB now #servicesInDB = Utils.unpack(self.rsClient.getService()) servicesInDB = self.rsClient.getService() if not servicesInDB[ 'OK' ]: gLogger.error( servicesInDB[ 'Message' ] ) return servicesInDB servicesInDB = servicesInDB[ 'Value' ] for service_name, service_type, site_name in servicesInDB: if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]: #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []: resource = self.rsClient.getResource( siteName = site_name, serviceType = service_type ) if not resource[ 'OK' ]: gLogger.error( resource[ 'Message' ] ) return resource if resource[ 'Value' ] == []: gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif service_type == "Storage": res = self.rsClient.getSite( siteName = site_name, meta = { 'columns' : 'GridSiteName'} ) if res[ 'OK' ]: res = res[ 'Value' ] else: res = [] if res: if self.rsClient.getResource( gridSiteName = res[0], serviceType = service_type ) == []: gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def _syncRegistryUsers(self): users = CS.getTypedDictRootedAt("Users", root= "/Registry") usersInCS = set(users.keys()) #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache()))) usersInCache = self.rmClient.getUserRegistryCache() if not usersInCache[ 'OK' ]: gLogger.error( usersInCache[ 'Message' ] ) return usersInCache usersInDB = set( [ userInCache[ 0 ] for userInCache in usersInCache[ 'Value' ] ] ) usersToAdd = usersInCS - usersInDB usersToDel = usersInDB - usersInCS gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel))) if len(usersToAdd) > 0: gLogger.debug(str(usersToAdd)) if len(usersToDel) > 0: gLogger.debug(str(usersToDel)) for u in usersToAdd: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower())) res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower() ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res for u in usersToDel: #Utils.protect2(self.rmClient.deleteUserRegistryCache, u) res = self.rmClient.deleteUserRegistryCache( u ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF