class Synchronizer( object ): def __init__( self, rsClient = None, rmClient = None ): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient() if rsClient == None else rsClient self.rmClient = ResourceManagementClient() if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ] ################################################################################ def sync( self, _a, _b ): """ :params: :attr:`thingsToSync`: list of things to sync """ gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ", ".join(self.synclist) ) ) for thing in self.synclist: getattr( self, '_sync' + thing )() return S_OK() ################################################################################ def __purge_resource( self, resourceName ): # Maybe remove attached SEs #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName)) SEs = self.rsClient.getStorageElement( resourceName = resourceName ) if not SEs[ 'OK' ]: gLogger.error( SEs[ 'Message' ] ) return SEs #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs])) SEs = [ se[0] for se in SEs ] res = self.rsClient.removeElement( 'StorageElement', SEs ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Remove resource itself. #Utils.unpack(self.rsClient.removeElement("Resource", resourceName)) res = self.rsClient.removeElement( 'Resource', resourceName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def __purge_site( self, siteName ): # Remove associated resources and services #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName)) resources = self.rsClient.getResource( siteName = siteName ) if not resources[ 'OK' ]: gLogger.error( resources[ 'Message' ] ) return resources #services = Utils.unpack(self.rsClient.getService(siteName=siteName)) services = self.rsClient.getService( siteName = siteName ) if not services[ 'OK' ]: gLogger.error( services[ 'Message' ] ) return services #_ = [self.__purge_resource(r[0]) for r in resources] for resource in resources: res = self.__purge_resource( resource[ 0 ] ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services])) services = [ service[ 0 ] for service in services[ 'Value' ] ] res = self.rsClient.removeElement( 'Service', services ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Remove site itself #Utils.unpack(self.rsClient.removeElement("Site", siteName)) res = self.rsClient.removeElement( 'Site', siteName ) if not res[ 'OK' ]: gLogger.info( res[ 'Message' ] ) return res def _syncSites( self ): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites[ 'OK' ]: gLogger.error( sites[ 'Message' ] ) return sites sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] ) # sites in CS now sitesCS = set( CS.getSites() ) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier( site )) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName( site ) if not gridSiteName[ 'OK' ]: gLogger.error( gridSiteName[ 'Message' ] ) return gridSiteName gridSiteName = gridSiteName[ 'Value' ] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites[ 'OK' ]: gLogger.error( DIRACSitesOfGridSites[ 'Message' ] ) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ] if len( DIRACSitesOfGridSites ) == 1: gt = tier else: gt = getGOCTier( DIRACSitesOfGridSites ) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite( gridSiteName, gt ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite( site, tier, gridSiteName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite( site, tier, "NULL" ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ # _syncResources HELPER functions def __updateService(self, site, type_): service = type_ + '@' + site #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site ) res = self.rsClient.addOrModifyService( service, type_, site ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def __getServiceEndpointInfo(self, node): #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) ) res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) if res['OK']: res = res[ 'Value' ] else: gLogger.warn( 'Error getting hostname info for %s' % node ) return [] if res == []: #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) ) url = Utils.canonicalURL(node) res = self.GOCDBClient.getServiceEndpointInfo('hostname', url ) if res['OK']: res = res[ 'Value' ] else: gLogger.warn( 'Error getting canonical hostname info for %s' % node ) res = [] return res def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] ) if not siteName[ 'OK' ]: gLogger.error( siteName[ 'Message' ] ) return siteName sites.append( siteName[ 'Value' ] ) sites = Utils.list_sanitize( Utils.list_flatten( sites ) ) _ = [ self.__updateService(s, serviceType) for s in sites ] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert(type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res resourcesInDB.add( node ) ################################################################################ def _syncResources( self ): gLogger.info("Starting sync of Resources") # resources in the DB now #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource()))) resources = self.rsClient.getResource() if not resources[ 'OK' ]: gLogger.error( resources[ 'Message' ] ) return resources resourcesInDB = set( [ resource[ 0 ] for resource in resources[ 'Value' ] ] ) # Site-CE / Site-SE mapping in CS now #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' )) CEinCS = getSiteCEMapping( 'LCG' ) if not CEinCS[ 'OK' ]: gLogger.error( CEinCS[ 'Message' ] ) return CEinCS CEinCS = CEinCS[ 'Value' ] # All CEs in CS now CEInCS = Utils.set_sanitize([CE for celist in CEinCS.values() for CE in celist]) # All SE Nodes in CS now SENodeInCS = set(CS.getSENodes()) # LFC Nodes in CS now LFCNodeInCS_L = set(CS.getLFCNode(readable = "ReadOnly")) LFCNodeInCS_C = set(CS.getLFCNode(readable = "ReadWrite")) # FTS Nodes in CS now FTSNodeInCS = set([v.split("/")[2][0:-5] for v in CS.getTypedDictRootedAt(root="/Resources/FTSEndpoints").values()]) # VOMS Nodes in CS now VOMSNodeInCS = set(CS.getVOMSEndpoints()) # complete list of resources in CS now resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS gLogger.info(" %d resources in CS, %s resources in DB, updating %d resources" % (len(resourcesInCS), len(resourcesInDB), len(resourcesInCS)-len(resourcesInDB))) # Remove resources that are not in the CS anymore for res in resourcesInDB - resourcesInCS: gLogger.info("Purging resource %s. Reason: not in CS anywore." % res) self.__purge_resource(res) # Add to DB what is in CS now and wasn't before # CEs for site in CEinCS: self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site) # SRMs self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage") # LFC_C self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage") # LFC_L self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage") # FTSs self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage") # VOMSs self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS") ################################################################################ def _syncStorageElements( self ): # Get StorageElements from the CS and the DB CSSEs = set(CS.getSEs()) #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement()))) ses = self.rsClient.getStorageElement() if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) return ses DBSEs = set( [ se[0] for se in ses[ 'Value' ] ] ) # Remove storageElements that are in DB but not in CS for se in DBSEs - CSSEs: #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se ) res = self.rsClient.removeElement( 'StorageElement', se ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Add new storage elements gLogger.info("Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs))) for SE in CSSEs - DBSEs: srm = CS.getSEHost( SE ) if not srm: gLogger.warn("%s has no srm URL in CS!!!" % SE) continue #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )) siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if siteInGOCDB[ 'OK' ]: siteInGOCDB = siteInGOCDB[ 'Value' ] else: gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm) continue if siteInGOCDB == []: gLogger.warn("%s is not in GOCDB!!!" % srm) continue siteInGOCDB = siteInGOCDB[ 0 ][ 'SITENAME' ] #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB ) res = self.rsClient.addOrModifyStorageElement( SE, srm, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ def _syncServices(self): """This function is in charge of cleaning the Service table in DB in case of obsolescence.""" # services in the DB now #servicesInDB = Utils.unpack(self.rsClient.getService()) servicesInDB = self.rsClient.getService() if not servicesInDB[ 'OK' ]: gLogger.error( servicesInDB[ 'Message' ] ) return servicesInDB servicesInDB = servicesInDB[ 'Value' ] for service_name, service_type, site_name in servicesInDB: if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]: #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []: resource = self.rsClient.getResource( siteName = site_name, serviceType = service_type ) if not resource[ 'OK' ]: gLogger.error( resource[ 'Message' ] ) return resource if resource[ 'Value' ] == []: gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif service_type == "Storage": res = self.rsClient.getSite( siteName = site_name, meta = { 'columns' : 'GridSiteName'} ) if res[ 'OK' ]: res = res[ 'Value' ] else: res = [] if res: if self.rsClient.getResource( gridSiteName = res[0], serviceType = service_type ) == []: gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def _syncRegistryUsers(self): users = CS.getTypedDictRootedAt("Users", root= "/Registry") usersInCS = set(users.keys()) #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache()))) usersInCache = self.rmClient.getUserRegistryCache() if not usersInCache[ 'OK' ]: gLogger.error( usersInCache[ 'Message' ] ) return usersInCache usersInDB = set( [ userInCache[ 0 ] for userInCache in usersInCache[ 'Value' ] ] ) usersToAdd = usersInCS - usersInDB usersToDel = usersInDB - usersInCS gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel))) if len(usersToAdd) > 0: gLogger.debug(str(usersToAdd)) if len(usersToDel) > 0: gLogger.debug(str(usersToDel)) for u in usersToAdd: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower())) res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower() ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res for u in usersToDel: #Utils.protect2(self.rmClient.deleteUserRegistryCache, u) res = self.rmClient.deleteUserRegistryCache( u ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class Synchronizer(object): def __init__(self, rsClient=None, rmClient=None): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient( ) if rsClient == None else rsClient self.rmClient = ResourceManagementClient( ) if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ] ################################################################################ def sync(self, _a, _b): """ :params: :attr:`thingsToSync`: list of things to sync """ gLogger.info("!!! Sync DB content with CS content for %s !!!" % (", ".join(self.synclist))) for thing in self.synclist: getattr(self, '_sync' + thing)() return S_OK() ################################################################################ def __purge_resource(self, resourceName): # Maybe remove attached SEs #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName)) SEs = self.rsClient.getStorageElement(resourceName=resourceName) if not SEs['OK']: gLogger.error(SEs['Message']) return SEs #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs])) SEs = [se[0] for se in SEs] res = self.rsClient.removeElement('StorageElement', SEs) if not res['OK']: gLogger.error(res['Message']) return res # Remove resource itself. #Utils.unpack(self.rsClient.removeElement("Resource", resourceName)) res = self.rsClient.removeElement('Resource', resourceName) if not res['OK']: gLogger.error(res['Message']) return res def __purge_site(self, siteName): # Remove associated resources and services #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName)) resources = self.rsClient.getResource(siteName=siteName) if not resources['OK']: gLogger.error(resources['Message']) return resources #services = Utils.unpack(self.rsClient.getService(siteName=siteName)) services = self.rsClient.getService(siteName=siteName) if not services['OK']: gLogger.error(services['Message']) return services #_ = [self.__purge_resource(r[0]) for r in resources] for resource in resources: res = self.__purge_resource(resource[0]) if not res['OK']: gLogger.error(res['Message']) return res #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services])) services = [service[0] for service in services['Value']] res = self.rsClient.removeElement('Service', services) if not res['OK']: gLogger.error(res['Message']) return res # Remove site itself #Utils.unpack(self.rsClient.removeElement("Site", siteName)) res = self.rsClient.removeElement('Site', siteName) if not res['OK']: gLogger.info(res['Message']) return res def _syncSites(self): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites['OK']: gLogger.error(sites['Message']) return sites sitesDB = set([site[0] for site in sites['Value']]) # sites in CS now sitesCS = set(CS.getSites()) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier(site)) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName(site) if not gridSiteName['OK']: gLogger.error(gridSiteName['Message']) return gridSiteName gridSiteName = gridSiteName['Value'] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName) if not DIRACSitesOfGridSites['OK']: gLogger.error(DIRACSitesOfGridSites['Message']) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len(DIRACSitesOfGridSites) == 1: gt = tier else: gt = getGOCTier(DIRACSitesOfGridSites) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite(gridSiteName, gt) if not res['OK']: gLogger.error(res['Message']) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite(site, tier, gridSiteName) if not res['OK']: gLogger.error(res['Message']) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite(site, tier, "NULL") if not res['OK']: gLogger.error(res['Message']) return res ################################################################################ # _syncResources HELPER functions def __updateService(self, site, type_): service = type_ + '@' + site #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site ) res = self.rsClient.addOrModifyService(service, type_, site) if not res['OK']: gLogger.error(res['Message']) return res def __getServiceEndpointInfo(self, node): #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) ) res = self.GOCDBClient.getServiceEndpointInfo('hostname', node) if res['OK']: res = res['Value'] else: gLogger.warn('Error getting hostname info for %s' % node) return [] if res == []: #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) ) url = Utils.canonicalURL(node) res = self.GOCDBClient.getServiceEndpointInfo('hostname', url) if res['OK']: res = res['Value'] else: gLogger.warn('Error getting canonical hostname info for %s' % node) res = [] return res def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site="NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [ self.__getServiceEndpointInfo(node) for node in nodesToUpdate ] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME']) if not siteName['OK']: gLogger.error(siteName['Message']) return siteName sites.append(siteName['Value']) sites = Utils.list_sanitize(Utils.list_flatten(sites)) _ = [self.__updateService(s, serviceType) for s in sites] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo( node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn( "Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert (type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB) if not res['OK']: gLogger.error(res['Message']) return res resourcesInDB.add(node) ################################################################################ def _syncResources(self): gLogger.info("Starting sync of Resources") # resources in the DB now #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource()))) resources = self.rsClient.getResource() if not resources['OK']: gLogger.error(resources['Message']) return resources resourcesInDB = set([resource[0] for resource in resources['Value']]) # Site-CE / Site-SE mapping in CS now #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' )) CEinCS = getSiteCEMapping('LCG') if not CEinCS['OK']: gLogger.error(CEinCS['Message']) return CEinCS CEinCS = CEinCS['Value'] # All CEs in CS now CEInCS = Utils.set_sanitize( [CE for celist in CEinCS.values() for CE in celist]) # All SE Nodes in CS now SENodeInCS = set(CS.getSENodes()) # LFC Nodes in CS now LFCNodeInCS_L = set(CS.getLFCNode(readable="ReadOnly")) LFCNodeInCS_C = set(CS.getLFCNode(readable="ReadWrite")) # FTS Nodes in CS now FTSNodeInCS = set([ v.split("/")[2][0:-5] for v in CS.getTypedDictRootedAt( root="/Resources/FTSEndpoints").values() ]) # VOMS Nodes in CS now VOMSNodeInCS = set(CS.getVOMSEndpoints()) # complete list of resources in CS now resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS gLogger.info( " %d resources in CS, %s resources in DB, updating %d resources" % (len(resourcesInCS), len(resourcesInDB), len(resourcesInCS) - len(resourcesInDB))) # Remove resources that are not in the CS anymore for res in resourcesInDB - resourcesInCS: gLogger.info("Purging resource %s. Reason: not in CS anywore." % res) self.__purge_resource(res) # Add to DB what is in CS now and wasn't before # CEs for site in CEinCS: self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site) # SRMs self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage") # LFC_C self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage") # LFC_L self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage") # FTSs self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage") # VOMSs self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS") ################################################################################ def _syncStorageElements(self): # Get StorageElements from the CS and the DB CSSEs = set(CS.getSEs()) #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement()))) ses = self.rsClient.getStorageElement() if not ses['OK']: gLogger.error(ses['Message']) return ses DBSEs = set([se[0] for se in ses['Value']]) # Remove storageElements that are in DB but not in CS for se in DBSEs - CSSEs: #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se ) res = self.rsClient.removeElement('StorageElement', se) if not res['OK']: gLogger.error(res['Message']) return res # Add new storage elements gLogger.info( "Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs))) for SE in CSSEs - DBSEs: srm = CS.getSEHost(SE) if not srm: gLogger.warn("%s has no srm URL in CS!!!" % SE) continue #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )) siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if siteInGOCDB['OK']: siteInGOCDB = siteInGOCDB['Value'] else: gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm) continue if siteInGOCDB == []: gLogger.warn("%s is not in GOCDB!!!" % srm) continue siteInGOCDB = siteInGOCDB[0]['SITENAME'] #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB ) res = self.rsClient.addOrModifyStorageElement(SE, srm, siteInGOCDB) if not res['OK']: gLogger.error(res['Message']) return res ################################################################################ def _syncServices(self): """This function is in charge of cleaning the Service table in DB in case of obsolescence.""" # services in the DB now #servicesInDB = Utils.unpack(self.rsClient.getService()) servicesInDB = self.rsClient.getService() if not servicesInDB['OK']: gLogger.error(servicesInDB['Message']) return servicesInDB servicesInDB = servicesInDB['Value'] for service_name, service_type, site_name in servicesInDB: if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]: #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []: resource = self.rsClient.getResource(siteName=site_name, serviceType=service_type) if not resource['OK']: gLogger.error(resource['Message']) return resource if resource['Value'] == []: gLogger.info( "Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement("Service", service_name) if not res['OK']: gLogger.error(res['Message']) return res elif service_type == "Storage": res = self.rsClient.getSite(siteName=site_name, meta={'columns': 'GridSiteName'}) if res['OK']: res = res['Value'] else: res = [] if res: if self.rsClient.getResource( gridSiteName=res[0], serviceType=service_type) == []: gLogger.info( "Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name) if not res['OK']: gLogger.error(res['Message']) return res def _syncRegistryUsers(self): users = CS.getTypedDictRootedAt("Users", root="/Registry") usersInCS = set(users.keys()) #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache()))) usersInCache = self.rmClient.getUserRegistryCache() if not usersInCache['OK']: gLogger.error(usersInCache['Message']) return usersInCache usersInDB = set( [userInCache[0] for userInCache in usersInCache['Value']]) usersToAdd = usersInCS - usersInDB usersToDel = usersInDB - usersInCS gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel))) if len(usersToAdd) > 0: gLogger.debug(str(usersToAdd)) if len(usersToDel) > 0: gLogger.debug(str(usersToDel)) for u in usersToAdd: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower())) res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower()) if not res['OK']: gLogger.error(res['Message']) return res for u in usersToDel: #Utils.protect2(self.rmClient.deleteUserRegistryCache, u) res = self.rmClient.deleteUserRegistryCache(u) if not res['OK']: gLogger.error(res['Message']) return res ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class GOCDB2CSAgent(AgentModule): """ Class to retrieve information about service endpoints from GOCDB and update configuration stored by CS """ def __init__(self, *args, **kwargs): """ c'tor """ super(GOCDB2CSAgent, self).__init__(*args, **kwargs) self.GOCDBClient = None self.csAPI = None self.dryRun = False def initialize(self): """ Run at the agent initialization (normally every 500 cycles) """ # client to connect to GOCDB self.GOCDBClient = GOCDBClient() self.dryRun = self.am_getOption('DryRun', self.dryRun) # API needed to update configuration stored by CS self.csAPI = CSAPI() return self.csAPI.initialize() def execute(self): """ Execute GOCDB queries according to the function map and user request (options in configuration). """ # __functionMap is at the end of the class definition for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems(): optionValue = self.am_getOption(option, True) if optionValue: result = functionCall(self) if not result['OK']: self.log.error("%s() failed with message: %s" % (functionCall.__name__, result['Message'])) else: self.log.info("Successfully executed %s" % functionCall.__name__) return S_OK() def updatePerfSONARConfiguration(self): """ Get current status of perfSONAR endpoints from GOCDB and update CS configuration accordingly. """ log = self.log.getSubLogger('updatePerfSONAREndpoints') log.debug('Begin function ...') # get endpoints result = self.__getPerfSONAREndpoints() if not result['OK']: log.error("__getPerfSONAREndpoints() failed with message: %s" % result['Message']) return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.') endpointList = result['Value'] # add DIRAC site name result = self.__addDIRACSiteName(endpointList) if not result['OK']: log.error("__addDIRACSiteName() failed with message: %s" % result['Message']) return S_ERROR('Unable to extend the list with DIRAC site names.') extendedEndpointList = result['Value'] # prepare dictionary with new configuration result = self.__preparePerfSONARConfiguration(extendedEndpointList) if not result['OK']: log.error( "__preparePerfSONARConfiguration() failed with message: %s" % result['Message']) return S_ERROR('Unable to prepare a new perfSONAR configuration.') finalConfiguration = result['Value'] # update configuration according to the final status of endpoints self.__updateConfiguration(finalConfiguration) log.debug("Configuration updated succesfully") log.debug('End function.') return S_OK() def __getPerfSONAREndpoints(self): """ Retrieve perfSONAR endpoint information directly from GOCDB. :return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB. """ log = self.log.getSubLogger('__getPerfSONAREndpoints') log.debug('Begin function ...') # get perfSONAR endpoints (latency and bandwidth) form GOCDB endpointList = [] for endpointType in ['Latency', 'Bandwidth']: result = self.GOCDBClient.getServiceEndpointInfo( 'service_type', 'net.perfSONAR.%s' % endpointType) if not result['OK']: log.error("getServiceEndpointInfo() failed with message: %s" % result['Message']) return S_ERROR('Could not fetch %s endpoints from GOCDB' % endpointType.lower()) log.debug('Number of %s endpoints: %s' % (endpointType.lower(), len(result['Value']))) endpointList.extend(result['Value']) log.debug('Number of perfSONAR endpoints: %s' % len(endpointList)) log.debug('End function.') return S_OK(endpointList) def __preparePerfSONARConfiguration(self, endpointList): """ Prepare a dictionary with a new CS configuration of perfSONAR endpoints. :return: Dictionary where keys are configuration paths (options and sections) and values are values of corresponding options or None in case of a path pointing to a section. """ log = self.log.getSubLogger('__preparePerfSONARConfiguration') log.debug('Begin function ...') # static elements of a path rootPath = '/Resources/Sites' extPath = 'Network' baseOptionName = 'Enabled' options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'} # enable GOCDB endpoints in configuration newConfiguration = {} for endpoint in endpointList: if endpoint['DIRACSITENAME'] is None: continue split = endpoint['DIRACSITENAME'].split('.') path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME']) for name, defaultValue in options.iteritems(): newConfiguration[cfgPath(path, name)] = defaultValue # get current configuration currentConfiguration = {} for option in options.iterkeys(): result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option) if not result['OK']: log.error("getConfigurationTree() failed with message: %s" % result['Message']) return S_ERROR('Unable to fetch perfSONAR endpoints from CS.') currentConfiguration.update(result['Value']) # disable endpoints that disappeared in GOCDB removedElements = set(currentConfiguration) - set(newConfiguration) newElements = set(newConfiguration) - set(currentConfiguration) addedEndpoints = len(newElements) / len(options) disabledEndpoints = 0 for path in removedElements: if baseOptionName in path: newConfiguration[path] = 'False' if currentConfiguration[path] != 'False': disabledEndpoints = disabledEndpoints + 1 # inform what will be changed if addedEndpoints > 0: self.log.info( "%s new perfSONAR endpoints will be added to the configuration" % addedEndpoints) if disabledEndpoints > 0: self.log.info( "%s old perfSONAR endpoints will be disable in the configuration" % disabledEndpoints) if addedEndpoints == 0 and disabledEndpoints == 0: self.log.info("perfSONAR configuration is up-to-date") log.debug('End function.') return S_OK(newConfiguration) def __addDIRACSiteName(self, inputList): """ Extend given list of GOCDB endpoints with DIRAC site name, i.e. add an entry "DIRACSITENAME" in dictionaries that describe endpoints. If given site name could not be found "DIRACSITENAME" is set to 'None'. :return: List of perfSONAR endpoints (dictionaries). """ log = self.log.getSubLogger('__addDIRACSiteName') log.debug('Begin function ...') # get site name dictionary result = getDIRACGOCDictionary() if not result['OK']: log.error("getDIRACGOCDictionary() failed with message: %s" % result['Message']) return S_ERROR('Could not get site name dictionary') # reverse the dictionary (assume 1 to 1 relation) DIRACGOCDict = result['Value'] GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys())) # add DIRAC site names outputList = [] for entry in inputList: try: entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']] except KeyError: self.log.warn("No dictionary entry for %s. " % entry['SITENAME']) entry['DIRACSITENAME'] = None outputList.append(entry) log.debug('End function.') return S_OK(outputList) def __updateConfiguration(self, setElements=None, delElements=None): """ Update configuration stored by CS. """ if setElements is None: setElements = {} if delElements is None: delElements = [] log = self.log.getSubLogger('__updateConfiguration') log.debug('Begin function ...') # assure existence and proper value of a section or an option for path, value in setElements.iteritems(): if value is None: section = path else: split = path.rsplit('/', 1) section = split[0] try: result = self.csAPI.createSection(section) if not result['OK']: log.error("createSection() failed with message: %s" % result['Message']) except Exception as e: log.error("Exception in createSection(): %s" % repr(e).replace(',)', ')')) if value is not None: try: result = self.csAPI.setOption(path, value) if not result['OK']: log.error("setOption() failed with message: %s" % result['Message']) except Exception as e: log.error("Exception in setOption(): %s" % repr(e).replace(',)', ')')) # delete elements in the configuration for path in delElements: result = self.csAPI.delOption(path) if not result['OK']: log.warn("delOption() failed with message: %s" % result['Message']) result = self.csAPI.delSection(path) if not result['OK']: log.warn("delSection() failed with message: %s" % result['Message']) if self.dryRun: log.info("Dry Run: CS won't be updated") self.csAPI.showDiff() else: # update configuration stored by CS result = self.csAPI.commit() if not result['OK']: log.error("commit() failed with message: %s" % result['Message']) return S_ERROR("Could not commit changes to CS.") else: log.info("Committed changes to CS") log.debug('End function.') return S_OK() # define mapping between an agent option in the configuration and a function call __functionMap = { 'UpdatePerfSONARS': updatePerfSONARConfiguration, }
class Synchronizer: ############################################################################# def __init__( self, rsDBin = None, rmDBin = None ): self.rsDB = rsDBin self.rmDB = rmDBin if self.rsDB == None and self.rmDB == None: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.GOCDBClient = GOCDBClient() ############################################################################# # def sync(self, thingsToSync = None, fake_param = None): def sync( self, _a, _b ): """ :params: :attr:`thingsToSync`: list of things to sync """ thingsToSync = ['Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers'] gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ' '.join( x for x in thingsToSync ) ) ) for thing in thingsToSync: getattr( self, '_sync' + thing )() return S_OK() ############################################################################# def _syncUtils( self ): """ Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils` """ statusIn = self.rsDB.getStatusList() #delete status not more in Utils for stIn in statusIn: if stIn not in ValidStatus: self.rsDB.removeStatus( stIn ) #Add new status for s in ValidStatus: if s not in statusIn: self.rsDB.addStatus( s ) for g in ( 'Site', 'Service', 'Resource' ): typeIn = self.rsDB.getTypesList( g ) if g == 'Site': typesList = ValidSiteType elif g == 'Service': typesList = ValidServiceType if g == 'Resource': typesList = ValidResourceType #delete types not more in Utils for tIn in typeIn: if tIn not in typesList: self.rsDB.removeType( g, tIn ) #Add new types for t in typesList: if t not in typeIn: self.rsDB.addType( g, t ) ############################################################################# def _syncSites( self ): """ Sync DB content with sites that are in the CS """ # sites in the DB now sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] ) sitesIn = [s[0] for s in sitesIn] # sites in CS now sitesList = getSites()['Value'] try: sitesList.remove( 'LCG.Dummy.ch' ) except ValueError: pass # remove sites from the DB not more in the CS for site in sitesIn: if site not in sitesList: self.rsDB.removeSite( site ) # add to DB what is in CS now and wasn't before for site in sitesList: if site not in sitesIn: # DIRAC Tier tier = getSiteTier( site )['Value'][0] if tier == 0 or tier == '0': t = 'T0' elif tier == 1 or tier == '1': t = 'T1' elif tier == 3 or tier == '3': t = 'T3' else: t = 'T2' #Grid Name of the site gridSiteName = getGOCSiteName( site ) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] #Grid Tier (with a workaround!) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites['OK']: raise RSSException, DIRACSitesOfGridSites['Message'] DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len( DIRACSitesOfGridSites ) == 1: gt = t else: gt = self.__getGOCTier( DIRACSitesOfGridSites ) self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) self.rsDB.addOrModifyGridSite( gridSiteName, gt ) sitesIn.append( site ) ############################################################################# def _syncVOBOX( self ): """ Sync DB content with VOBoxes """ # services in the DB now servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] ) servicesIn = [s[0] for s in servicesIn] for site in ['LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl']: service = 'VO-BOX@' + site if service not in servicesIn: self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) ############################################################################# def _syncResources( self ): # resources in the DB now resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] ) resourcesIn = [r[0] for r in resourcesIn] # services in the DB now servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] ) servicesIn = [s[0] for s in servicesIn] # Site-CE mapping in CS now siteCE = getSiteCEMapping( 'LCG' )['Value'] # Site-SE mapping in CS now siteSE = getSiteSEMapping( 'LCG' )['Value'] # CEs in CS now CEList = [] for i in siteCE.values(): for ce in i: if ce is None: continue CEList.append( ce ) # SEs in CS now SEList = [] for i in siteSE.values(): for x in i: SEList.append( x ) # SE Nodes in CS now SENodeList = [] for SE in SEList: node = getSENodes( SE )['Value'][0] if node is None: continue if node not in SENodeList: SENodeList.append( node ) # LFC Nodes in CS now LFCNodeList_L = [] LFCNodeList_C = [] for site in getLFCSites()['Value']: for readable in ( 'ReadOnly', 'ReadWrite' ): LFCNode = getLFCNode( site, readable )['Value'] if LFCNode is None or LFCNode == []: continue LFCNode = LFCNode[0] if readable == 'ReadWrite': if LFCNode not in LFCNodeList_C: LFCNodeList_C.append( LFCNode ) elif readable == 'ReadOnly': if LFCNode not in LFCNodeList_L: LFCNodeList_L.append( LFCNode ) # FTS Nodes in CS now FTSNodeList = [] sitesWithFTS = getFTSSites() for site in sitesWithFTS['Value']: fts = getFTSEndpoint( site )['Value'] if fts is None or fts == []: continue fts = fts[0] if fts not in FTSNodeList: FTSNodeList.append( fts ) # VOMS Nodes in CS now VOMSNodeList = getVOMSEndpoints()['Value'] # complete list of resources in CS now resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList # list of services in CS now (to be done) servicesList = [] #remove resources no more in the CS for res in resourcesIn: if res not in resourcesList: self.rsDB.removeResource( res ) self.rsDB.removeStorageElement( resourceName = res ) # add to DB what is in CS now and wasn't before # CEs for site in siteCE.keys(): if site == 'LCG.Dummy.ch': continue for ce in siteCE[site]: if ce is None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( ce )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue serviceType = 'Computing' service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if ce not in resourcesIn: CEType = getCEType( site, ce )['Value'] ceType = 'CE' if CEType == 'CREAM': ceType = 'CREAMCE' self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( ce ) # SRMs for srm in SENodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( srm )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if srm not in resourcesIn and srm is not None: self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( srm ) # LFC_C for lfc in LFCNodeList_C: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( lfc )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( lfc ) # LFC_L for lfc in LFCNodeList_L: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( lfc )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( lfc ) # FTSs for fts in FTSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( fts )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if fts not in resourcesIn and fts is not None: self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( fts ) # VOMSs for voms in VOMSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( voms )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] site = siteInDIRAC['Value'] serviceType = 'VOMS' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if voms not in resourcesIn and voms is not None: self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( voms ) #remove services no more in the CS for ser in servicesIn: if ser not in servicesList: serType = ser.split( '@' )[0] if serType != 'VO-BOX': self.rsDB.removeService( ser ) self.rsDB.removeResource( serviceName = ser ) site = ser.split( '@' )[1] if serType == 'Storage': self.rsDB.removeStorageElement( siteName = site ) ############################################################################# def _syncStorageElements( self ): # Get StorageElements from the CS SEs = getStorageElements() if not SEs['OK']: raise RSSException, SEs['Message'] SEs = SEs['Value'] for access in ( 'Read', 'Write' ): storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access, paramsList = [ 'StorageElementName' ] ) try: storageElementsIn = [ x[ 0 ] for x in storageElementsIn ] except IndexError: pass #remove storageElements no more in the CS for se in storageElementsIn: if se not in SEs: self.rsDB.removeStorageElement( storageElementName = se, resourceName = None, access = access ) #Add new storage Elements for SE in SEs: srm = getSENodes( SE )[ 'Value' ][ 0 ] if srm == None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if not siteInGOCDB[ 'OK' ]: raise RSSException, siteInGOCDB[ 'Message' ] if siteInGOCDB[ 'Value' ] == []: continue siteInGOCDB = siteInGOCDB[ 'Value' ][ 0 ][ 'SITENAME' ] if SE not in storageElementsIn: self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ), access = access ) storageElementsIn.append( SE ) ############################################################################# def __getGOCTier( self, sitesList ): gridTier = 3 for site in sitesList: tier = getSiteTier( site )['Value'][0] if tier == 0 or tier == '0': tn = 0 elif tier == 1 or tier == '1': tn = 1 elif tier == 3 or tier == '3': tn = 3 else: tn = 2 if tn < gridTier: gridTier = tn if gridTier == 0: gt = 'T0' elif gridTier == 1: gt = 'T1' elif gridTier == 3: gt = 'T3' else: gt = 'T2' return gt ############################################################################# def _syncRegistryUsers(self): from DIRAC.ResourceStatusSystem.Utilities import CS users = CS.getTypedDictRootedAt("Users", root= "/Registry") for u in users: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())
class GOCDB2CSAgent (AgentModule): """ Class to retrieve information about service endpoints from GOCDB and update configuration stored by CS """ def __init__(self, *args, **kwargs): """ c'tor """ super(GOCDB2CSAgent, self).__init__(*args, **kwargs) self.GOCDBClient = None self.csAPI = None self.dryRun = False def initialize(self): """ Run at the agent initialization (normally every 500 cycles) """ # client to connect to GOCDB self.GOCDBClient = GOCDBClient() self.dryRun = self.am_getOption('DryRun', self.dryRun) # API needed to update configuration stored by CS self.csAPI = CSAPI() return self.csAPI.initialize() def execute(self): """ Execute GOCDB queries according to the function map and user request (options in configuration). """ # __functionMap is at the end of the class definition for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems(): optionValue = self.am_getOption(option, True) if optionValue: result = functionCall(self) if not result['OK']: self.log.error("%s() failed with message: %s" % (functionCall.__name__, result['Message'])) else: self.log.info("Successfully executed %s" % functionCall.__name__) return S_OK() def updatePerfSONARConfiguration(self): """ Get current status of perfSONAR endpoints from GOCDB and update CS configuration accordingly. """ log = self.log.getSubLogger('updatePerfSONAREndpoints') log.debug('Begin function ...') # get endpoints result = self.__getPerfSONAREndpoints() if not result['OK']: log.error("__getPerfSONAREndpoints() failed with message: %s" % result['Message']) return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.') endpointList = result['Value'] # add DIRAC site name result = self.__addDIRACSiteName(endpointList) if not result['OK']: log.error("__addDIRACSiteName() failed with message: %s" % result['Message']) return S_ERROR('Unable to extend the list with DIRAC site names.') extendedEndpointList = result['Value'] # prepare dictionary with new configuration result = self.__preparePerfSONARConfiguration(extendedEndpointList) if not result['OK']: log.error("__preparePerfSONARConfiguration() failed with message: %s" % result['Message']) return S_ERROR('Unable to prepare a new perfSONAR configuration.') finalConfiguration = result['Value'] # update configuration according to the final status of endpoints self.__updateConfiguration(finalConfiguration) log.debug("Configuration updated succesfully") log.debug('End function.') return S_OK() def __getPerfSONAREndpoints(self): """ Retrieve perfSONAR endpoint information directly from GOCDB. :return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB. """ log = self.log.getSubLogger('__getPerfSONAREndpoints') log.debug('Begin function ...') # get perfSONAR endpoints (latency and bandwidth) form GOCDB endpointList = [] for endpointType in ['Latency', 'Bandwidth']: result = self.GOCDBClient.getServiceEndpointInfo('service_type', 'net.perfSONAR.%s' % endpointType) if not result['OK']: log.error("getServiceEndpointInfo() failed with message: %s" % result['Message']) return S_ERROR('Could not fetch %s endpoints from GOCDB' % endpointType.lower()) log.debug('Number of %s endpoints: %s' % (endpointType.lower(), len(result['Value']))) endpointList.extend(result['Value']) log.debug('Number of perfSONAR endpoints: %s' % len(endpointList)) log.debug('End function.') return S_OK(endpointList) def __preparePerfSONARConfiguration(self, endpointList): """ Prepare a dictionary with a new CS configuration of perfSONAR endpoints. :return: Dictionary where keys are configuration paths (options and sections) and values are values of corresponding options or None in case of a path pointing to a section. """ log = self.log.getSubLogger('__preparePerfSONARConfiguration') log.debug('Begin function ...') # static elements of a path rootPath = '/Resources/Sites' extPath = 'Network' baseOptionName = 'Enabled' options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'} # enable GOCDB endpoints in configuration newConfiguration = {} for endpoint in endpointList: if endpoint['DIRACSITENAME'] is None: continue split = endpoint['DIRACSITENAME'].split('.') path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME']) for name, defaultValue in options.iteritems(): newConfiguration[cfgPath(path, name)] = defaultValue # get current configuration currentConfiguration = {} for option in options.iterkeys(): result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option) if not result['OK']: log.error("getConfigurationTree() failed with message: %s" % result['Message']) return S_ERROR('Unable to fetch perfSONAR endpoints from CS.') currentConfiguration.update(result['Value']) # disable endpoints that disappeared in GOCDB removedElements = set(currentConfiguration) - set(newConfiguration) newElements = set(newConfiguration) - set(currentConfiguration) addedEndpoints = len(newElements) / len(options) disabledEndpoints = 0 for path in removedElements: if baseOptionName in path: newConfiguration[path] = 'False' if currentConfiguration[path] != 'False': disabledEndpoints = disabledEndpoints + 1 # inform what will be changed if addedEndpoints > 0: self.log.info("%s new perfSONAR endpoints will be added to the configuration" % addedEndpoints) if disabledEndpoints > 0: self.log.info("%s old perfSONAR endpoints will be disable in the configuration" % disabledEndpoints) if addedEndpoints == 0 and disabledEndpoints == 0: self.log.info("perfSONAR configuration is up-to-date") log.debug('End function.') return S_OK(newConfiguration) def __addDIRACSiteName(self, inputList): """ Extend given list of GOCDB endpoints with DIRAC site name, i.e. add an entry "DIRACSITENAME" in dictionaries that describe endpoints. If given site name could not be found "DIRACSITENAME" is set to 'None'. :return: List of perfSONAR endpoints (dictionaries). """ log = self.log.getSubLogger('__addDIRACSiteName') log.debug('Begin function ...') # get site name dictionary result = getDIRACGOCDictionary() if not result['OK']: log.error("getDIRACGOCDictionary() failed with message: %s" % result['Message']) return S_ERROR('Could not get site name dictionary') # reverse the dictionary (assume 1 to 1 relation) DIRACGOCDict = result['Value'] GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys())) # add DIRAC site names outputList = [] for entry in inputList: try: entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']] except KeyError: self.log.warn("No dictionary entry for %s. " % entry['SITENAME']) entry['DIRACSITENAME'] = None outputList.append(entry) log.debug('End function.') return S_OK(outputList) def __updateConfiguration(self, setElements=None, delElements=None): """ Update configuration stored by CS. """ if setElements is None: setElements = {} if delElements is None: delElements = [] log = self.log.getSubLogger('__updateConfiguration') log.debug('Begin function ...') # assure existence and proper value of a section or an option for path, value in setElements.iteritems(): if value is None: section = path else: split = path.rsplit('/', 1) section = split[0] try: result = self.csAPI.createSection(section) if not result['OK']: log.error("createSection() failed with message: %s" % result['Message']) except Exception as e: log.error("Exception in createSection(): %s" % repr(e).replace(',)', ')')) if value is not None: try: result = self.csAPI.setOption(path, value) if not result['OK']: log.error("setOption() failed with message: %s" % result['Message']) except Exception as e: log.error("Exception in setOption(): %s" % repr(e).replace(',)', ')')) # delete elements in the configuration for path in delElements: result = self.csAPI.delOption(path) if not result['OK']: log.warn("delOption() failed with message: %s" % result['Message']) result = self.csAPI.delSection(path) if not result['OK']: log.warn("delSection() failed with message: %s" % result['Message']) if self.dryRun: log.info("Dry Run: CS won't be updated") self.csAPI.showDiff() else: # update configuration stored by CS result = self.csAPI.commit() if not result['OK']: log.error("commit() failed with message: %s" % result['Message']) return S_ERROR("Could not commit changes to CS.") else: log.info("Committed changes to CS") log.debug('End function.') return S_OK() # define mapping between an agent option in the configuration and a function call __functionMap = {'UpdatePerfSONARS': updatePerfSONARConfiguration, }
class Synchronizer: ############################################################################# def __init__(self, rsDBin=None, rmDBin=None): self.rsDB = rsDBin self.rmDB = rmDBin if self.rsDB == None and self.rmDB == None: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.GOCDBClient = GOCDBClient() ############################################################################# # def sync(self, thingsToSync = None, fake_param = None): def sync(self, _a, _b): """ :params: :attr:`thingsToSync`: list of things to sync """ thingsToSync = [ 'Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers' ] gLogger.info("!!! Sync DB content with CS content for %s !!!" % (' '.join(x for x in thingsToSync))) for thing in thingsToSync: getattr(self, '_sync' + thing)() return S_OK() ############################################################################# def _syncUtils(self): """ Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils` """ statusIn = self.rsDB.getStatusList() #delete status not more in Utils for stIn in statusIn: if stIn not in ValidStatus: self.rsDB.removeStatus(stIn) #Add new status for s in ValidStatus: if s not in statusIn: self.rsDB.addStatus(s) for g in ('Site', 'Service', 'Resource'): typeIn = self.rsDB.getTypesList(g) if g == 'Site': typesList = ValidSiteType elif g == 'Service': typesList = ValidServiceType if g == 'Resource': typesList = ValidResourceType #delete types not more in Utils for tIn in typeIn: if tIn not in typesList: self.rsDB.removeType(g, tIn) #Add new types for t in typesList: if t not in typeIn: self.rsDB.addType(g, t) ############################################################################# def _syncSites(self): """ Sync DB content with sites that are in the CS """ # sites in the DB now sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName']) sitesIn = [s[0] for s in sitesIn] # sites in CS now sitesList = getSites()['Value'] try: sitesList.remove('LCG.Dummy.ch') except ValueError: pass # remove sites from the DB not more in the CS for site in sitesIn: if site not in sitesList: self.rsDB.removeSite(site) # add to DB what is in CS now and wasn't before for site in sitesList: if site not in sitesIn: # DIRAC Tier tier = getSiteTier(site)['Value'][0] if tier == 0 or tier == '0': t = 'T0' elif tier == 1 or tier == '1': t = 'T1' elif tier == 3 or tier == '3': t = 'T3' else: t = 'T2' #Grid Name of the site gridSiteName = getGOCSiteName(site) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] #Grid Tier (with a workaround!) DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName) if not DIRACSitesOfGridSites['OK']: raise RSSException, DIRACSitesOfGridSites['Message'] DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len(DIRACSitesOfGridSites) == 1: gt = t else: gt = self.__getGOCTier(DIRACSitesOfGridSites) self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) self.rsDB.addOrModifyGridSite(gridSiteName, gt) sitesIn.append(site) ############################################################################# def _syncVOBOX(self): """ Sync DB content with VOBoxes """ # services in the DB now servicesIn = self.rsDB.getMonitoredsList('Service', paramsList=['ServiceName']) servicesIn = [s[0] for s in servicesIn] for site in [ 'LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl' ]: service = 'VO-BOX@' + site if service not in servicesIn: self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) ############################################################################# def _syncResources(self): # resources in the DB now resourcesIn = self.rsDB.getMonitoredsList('Resource', paramsList=['ResourceName']) resourcesIn = [r[0] for r in resourcesIn] # services in the DB now servicesIn = self.rsDB.getMonitoredsList('Service', paramsList=['ServiceName']) servicesIn = [s[0] for s in servicesIn] # Site-CE mapping in CS now siteCE = getSiteCEMapping('LCG')['Value'] # Site-SE mapping in CS now siteSE = getSiteSEMapping('LCG')['Value'] # CEs in CS now CEList = [] for i in siteCE.values(): for ce in i: if ce is None: continue CEList.append(ce) # SEs in CS now SEList = [] for i in siteSE.values(): for x in i: SEList.append(x) # SE Nodes in CS now SENodeList = [] for SE in SEList: node = getSENodes(SE)['Value'][0] if node is None: continue if node not in SENodeList: SENodeList.append(node) # LFC Nodes in CS now LFCNodeList_L = [] LFCNodeList_C = [] for site in getLFCSites()['Value']: for readable in ('ReadOnly', 'ReadWrite'): LFCNode = getLFCNode(site, readable)['Value'] if LFCNode is None or LFCNode == []: continue LFCNode = LFCNode[0] if readable == 'ReadWrite': if LFCNode not in LFCNodeList_C: LFCNodeList_C.append(LFCNode) elif readable == 'ReadOnly': if LFCNode not in LFCNodeList_L: LFCNodeList_L.append(LFCNode) # FTS Nodes in CS now FTSNodeList = [] sitesWithFTS = getFTSSites() for site in sitesWithFTS['Value']: fts = getFTSEndpoint(site)['Value'] if fts is None or fts == []: continue fts = fts[0] if fts not in FTSNodeList: FTSNodeList.append(fts) # VOMS Nodes in CS now VOMSNodeList = getVOMSEndpoints()['Value'] # complete list of resources in CS now resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList # list of services in CS now (to be done) servicesList = [] #remove resources no more in the CS for res in resourcesIn: if res not in resourcesList: self.rsDB.removeResource(res) self.rsDB.removeStorageElement(resourceName=res) # add to DB what is in CS now and wasn't before # CEs for site in siteCE.keys(): if site == 'LCG.Dummy.ch': continue for ce in siteCE[site]: if ce is None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(ce)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue serviceType = 'Computing' service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if ce not in resourcesIn: CEType = getCEType(site, ce)['Value'] ceType = 'CE' if CEType == 'CREAM': ceType = 'CREAMCE' self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(ce) # SRMs for srm in SENodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(srm)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if srm not in resourcesIn and srm is not None: self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(srm) # LFC_C for lfc in LFCNodeList_C: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(lfc)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(lfc) # LFC_L for lfc in LFCNodeList_L: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(lfc)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(lfc) # FTSs for fts in FTSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(fts)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if fts not in resourcesIn and fts is not None: self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(fts) # VOMSs for voms in VOMSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(voms)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] site = siteInDIRAC['Value'] serviceType = 'VOMS' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if voms not in resourcesIn and voms is not None: self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(voms) #remove services no more in the CS for ser in servicesIn: if ser not in servicesList: serType = ser.split('@')[0] if serType != 'VO-BOX': self.rsDB.removeService(ser) self.rsDB.removeResource(serviceName=ser) site = ser.split('@')[1] if serType == 'Storage': self.rsDB.removeStorageElement(siteName=site) ############################################################################# def _syncStorageElements(self): # Get StorageElements from the CS SEs = getStorageElements() if not SEs['OK']: raise RSSException, SEs['Message'] SEs = SEs['Value'] for access in ('Read', 'Write'): storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access, paramsList=['StorageElementName']) try: storageElementsIn = [x[0] for x in storageElementsIn] except IndexError: pass #remove storageElements no more in the CS for se in storageElementsIn: if se not in SEs: self.rsDB.removeStorageElement(storageElementName=se, resourceName=None, access=access) #Add new storage Elements for SE in SEs: srm = getSENodes(SE)['Value'][0] if srm == None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: continue siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] if SE not in storageElementsIn: self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59), access=access) storageElementsIn.append(SE) ############################################################################# def __getGOCTier(self, sitesList): gridTier = 3 for site in sitesList: tier = getSiteTier(site)['Value'][0] if tier == 0 or tier == '0': tn = 0 elif tier == 1 or tier == '1': tn = 1 elif tier == 3 or tier == '3': tn = 3 else: tn = 2 if tn < gridTier: gridTier = tn if gridTier == 0: gt = 'T0' elif gridTier == 1: gt = 'T1' elif gridTier == 3: gt = 'T3' else: gt = 'T2' return gt ############################################################################# def _syncRegistryUsers(self): from DIRAC.ResourceStatusSystem.Utilities import CS users = CS.getTypedDictRootedAt("Users", root="/Registry") for u in users: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())
class AutoVac2CSAgent(AgentModule): """ AutoBdii2CSAgent. Automatically updates the CS automatically for CEs and SEs. """ max_cputime_map = {'VAC': 400000, 'CLOUD': 24000000} cc_regex = re.compile(r'\.([a-zA-Z]{2})$') cc_mappings = {'.gov': 'us', '.edu': 'us', 'efda.org': 'uk', 'atlas-swt2.org': 'us'} def initialize(self, *args, **kwargs): """ Initialize. Initialise method pulls in some extra configuration options These include: VOKeys - List of VO identifiers """ self.vokeys = self.am_getOption('VOKeys', ['GridPP']) self.removal_threshold = self.am_getOption('RemovalThreshold', 5) self.gocdb_client = GOCDBClient() return S_OK() def execute(self): """General agent execution method.""" cfg_system = ConfigurationSystem() cfg_system.initialize() # Get VAC sites. # ############## result = self.gocdb_client.getServiceEndpointInfo('service_type', "uk.ac.gridpp.vac") if not result['OK']: self.log.error("Problem getting GOCDB VAC information") return result try: self.process_gocdb_results(result['Value'], 'VAC', cfg_system) except: self.log.exception("Problem processing GOCDB VAC information") return S_ERROR("Problem processing GOCDB VAC information") # Get CLOUD (vcycle) sites. # ######################### result = self.gocdb_client.getServiceEndpointInfo('service_type', "uk.ac.gridpp.vcycle") if not result['OK']: self.log.error("Problem getting GOCDB CLOUD (vcycle) information") return result try: self.process_gocdb_results(result['Value'], 'CLOUD', cfg_system) except: self.log.exception("Problem processing GOCDB CLOUD (vcycle) information") return S_ERROR("Problem processing GOCDB CLOUD (vcycle) information") cfg_system.commit() # Remove old hosts/sites # ###################### try: self.remove_old(self.removal_threshold) except: self.log.exception("Problem removing old hosts/sites.") return S_ERROR("Problem processing GOCDB CLOUD (vcycle) information") return S_OK() def process_gocdb_results(self, services, site_path_prefix, cfg_system, country_default='xx'): """ Process GOCDB results. Args: services (list): List of services returned from GOCDB query. site_path_prefix (str): The CS path prefix (VAC or CLOUD) for the type of service that we are processing. cfg_system (ConfigurationSystem): A ConfigurationSystem instance used to update the CS. """ for service in services: # Resources sitename = service.get('SITENAME') hostname = service.get('HOSTNAME') country_code = AutoVac2CSAgent.extract_cc(hostname) or country_default if sitename is None or hostname is None: self.log.warn("Missing sitename or hostname for service:\n%s" % pformat(service)) continue site_path = cfgPath(SITES_BASE, site_path_prefix, "%s.%s.%s" % (site_path_prefix, sitename, country_code)) ce_path = cfgPath(site_path, 'CEs', hostname) queue_path = cfgPath(ce_path, 'Queues', 'default') cfg_system.add(site_path, 'Name', sitename) cfg_system.append_unique(site_path, 'CE', hostname) cfg_system.add(ce_path, 'CEType', site_path_prefix.capitalize()) cfg_system.add(ce_path, 'Architecture', 'x86_64') cfg_system.add(ce_path, 'OS', 'EL6') cfg_system.add(ce_path, 'LastSeen', date.today().strftime('%d/%m/%Y')) cfg_system.add(queue_path, 'maxCPUTime', AutoVac2CSAgent.max_cputime_map.get(site_path_prefix, 'Unknown')) for extension in service.get('EXTENSIONS', []): match = VOKEY_EXTENSION_REGEX.match(extension.get('KEY', '')) if match is None: continue extension_key = match.group() k, vokey = match.groups() if vokey not in self.vokeys: self.log.warn("Extension KEY %s for %s with vokey %s does not belong " "to a valid vokey: %s" % (extension_key, sitename, vokey, self.vokeys)) continue if k == 'SE': se = extension.get('VALUE') if se is None: self.log.warn("No SE value for extension KEY %s" % extension_key) continue cfg_system.append_unique(site_path, 'SE', se) # Registry elif k == 'DN': dn = extension.get('VALUE', '') if "CN=" not in dn: self.log.warn("For extension KEY %s, Could not find the CN component " "of DN: %s" % (extension_key, dn)) continue cn = max(CN_REGEX.findall(dn), key=len) host_path = cfgPath(HOSTS_BASE, cn) cfg_system.add(host_path, 'DN', dn) cfg_system.add(host_path, 'LastSeen', date.today().strftime('%d/%m/%Y')) cfg_system.add(host_path, 'Properties', ['GenericPilot', 'LimitedDelegation']) return S_OK() def remove_old(self, removal_threshold=5): """Remove old hosts/sites.""" cfg_system = ConfigurationSystem() result = cfg_system.getCurrentCFG() if not result['OK']: self.log.error('Could not get current config from the CS') raise RuntimeError("Error removing old Resources/Registry.") today = date.today() removal_threshold = timedelta(days=removal_threshold) old_ces = set() base_path = '/Resources/Sites' for site_type in ('VAC', 'CLOUD'): site_type_path = cfgPath(base_path, site_type) for site, site_info in result['Value'].getAsDict(base_path).iteritems(): site_path = cfgPath(site_type_path, site) for ce, ce_info in site_info.get('CEs', {}).iteritems(): ce_path = cfgPath(site_path, 'CEs', ce) if 'LastSeen' not in ce_info: self.log.warn("No LastSeen info for CE: %s at site: %s" % (ce, site)) continue last_seen = datetime.strptime(ce_info['LastSeen'], '%d/%m/%Y').date() delta = today - last_seen if delta > removal_threshold: self.log.warn("Last seen %s:%s %s days ago...removing" % (site, ce, delta.days)) cfg_system.remove(section=ce_path) old_ces.add(ce) if old_ces: cfg_system.remove(section=site_path, option='CE', value=old_ces) old_ces.clear() host_base = '/Registry/Hosts' for host, host_info in result['Value'].getAsDict(host_base).iteritems(): host_path = cfgPath(host_base, host) if 'LastSeen' not in host_info: self.log.warn("No LastSeen info for host: %s" % host) continue last_seen = datetime.strptime(host_info['LastSeen'], '%d/%m/%Y').date() delta = today - last_seen if delta > removal_threshold: self.log.warn("Last seen host %s %s days ago...removing" % (host, delta.days)) cfg_system.remove(section=host_path) cfg_system.commit() return S_OK() @classmethod def extract_cc(cls, ce, cc_mappings=None, cc_regex=None): """Extract the 2 character country code from the CE name.""" if cc_mappings is None: cc_mappings = cls.cc_mappings if cc_regex is None: cc_regex = cls.cc_regex ce = ce.strip().lower() for key, value in cc_mappings.iteritems(): if ce.endswith(key): return value cc = cc_regex.search(ce) if cc is not None: cc = cc.groups()[0] return cc