def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssFlag = ResourceStatus().rssFlag self.rsClient = ResourceStatusClient() cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache)
def __init__( self ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( 'StorageElement', cacheLifeTime, self.__updateSECache )
def __init__(self): """ Constructor. examples: >>> s = Synchronizer() """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.operations = Operations() self.resources = Resources() self.rStatus = ResourceStatusClient.ResourceStatusClient() self.rssConfig = RssConfiguration() self.diracAdmin = DiracAdmin()
def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() self.tokenOwner = "rs_svc" result = getProxyInfo() if result['OK']: self.tokenOwner = result['Value']['username']
def __init__( self, rssFlag = None ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = ResourceStatusClient() self.rssFlag = rssFlag if rssFlag is None: self.rssFlag = self.__getMode() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache( cacheLifeTime, self.__updateRssCache )
def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssFlag = ResourceStatus().rssFlag self.rsClient = ResourceStatusClient()
def __init__(self, rStatus=None, rManagement=None): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient.ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient.ResourceManagementClient() self.rssConfig = RssConfiguration()
def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() # this just sets the main owner, "rs_svc" just mean "RSS service" self.tokenOwner = "rs_svc" # if we are running this script as a user (from a CLI), # the username found the proxy will be used as tokenOwner result = getProxyInfo() if result['OK']: self.tokenOwner = result['Value']['username']
def __init__(self): """ Constructor, initializes the logger, rssClient and cache. examples >>> siteStatus = SiteStatus() """ super(SiteStatus, self).__init__() # RSSCache initialization cacheLifeTime = int(RssConfiguration().getConfigCache()) self.siteCache = RSSCache('Site', cacheLifeTime, self.__updateSiteCache)
def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssFlag = ResourceStatus().rssFlag self.rsClient = ResourceStatusClient() cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache)
def __init__( self ): """ Constructor, initializes the logger, rssClient and cache. examples >>> siteStatus = SiteStatus() """ super( SiteStatus, self ).__init__() # RSSCache initialization cacheLifeTime = int( RssConfiguration().getConfigCache() ) # FIXME: we need to define the types in the CS : Site => {Computing,Storage,..}Access self.siteCache = RSSCache( 'Site', cacheLifeTime, self.__updateSiteCache )
def __init__( self ): """ Constructor. examples: >>> s = Synchronizer() """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.operations = Operations() self.resources = Resources() self.rStatus = ResourceStatusClient.ResourceStatusClient() self.rssConfig = RssConfiguration() self.diracAdmin = DiracAdmin()
def __init__( self ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( 'StorageElement', cacheLifeTime, self.__updateSECache )
def __init__(self, rssFlag=None): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = ResourceStatusClient() self.rssFlag = rssFlag if rssFlag is None: self.rssFlag = self.__getMode() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache)
def __init__( self ): ''' Constructor, initializes the rssClient. ''' self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) cacheHistory = int( self.rssConfig.getConfigCacheHistory() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( cacheLifeTime, updateFunc = self.__updateSECache, cacheHistoryLifeTime = cacheHistory ) self.seCache.startRefreshThread()
def __init__( self ): """ Constructor, initializes the logger, rssClient and caches. examples >>> resourceStatus = ResourceStatus() """ super( ResourceStatus, self ).__init__() self.siteStatus = SiteStatus() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( RssConfiguration().getConfigCache() ) # RSSCaches, one per elementType ( StorageElement, ComputingElement ) # Should be generated on the fly, instead of being hardcoded ? self.seCache = RSSCache( 'Storage', cacheLifeTime, self._updateSECache ) self.ceCache = RSSCache( 'Computing', cacheLifeTime, self._updateCECache )
def __init__(self, lifeTime, updateFunc): """ Constructor :Parameters: **elementType** - `string` RSS elementType, e.g.: StorageElement, CE, Queue... note that one RSSCache can only hold elements of a single elementType to avoid issues while doing the Cartesian product. **lifeTime** - `int` Lifetime of the elements in the cache ( seconds ! ) **updateFunc** - `function` This function MUST return a S_OK | S_ERROR object. In the case of the first, its value must follow the dict format: ( key, value ) being key ( elementName, statusType ) and value status. """ super(RSSCache, self).__init__(lifeTime, updateFunc) self.allStatusTypes = RssConfiguration().getConfigStatusType()
class ResourceStatus( object ): ''' ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. ''' __metaclass__ = DIRACSingleton def __init__( self ): ''' Constructor, initializes the rssClient. ''' self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) cacheHistory = int( self.rssConfig.getConfigCacheHistory() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( cacheLifeTime, updateFunc = self.__updateSECache, cacheHistoryLifeTime = cacheHistory ) self.seCache.startRefreshThread() def getStorageElementStatus( self, elementName, statusType = None, default = None ): ''' Helper with dual access, tries to get information from the RSS for the given StorageElement, otherwise, it gets it from the CS. example: >>> getStorageElementStatus( 'CERN-USER', 'Read' ) S_OK( { 'CERN-USER' : { 'Read': 'Active' } } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( { 'CERN-USER' : {'Read': 'Active', 'Write': 'Active', 'Check': 'Banned', 'Remove': 'Banned'}} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) ''' if self.__getMode(): return self.__getRSSStorageElementStatus( elementName, statusType, default ) else: return self.__getCSStorageElementStatus( elementName, statusType, default ) def setStorageElementStatus( self, elementName, statusType, status, reason = None, tokenOwner = None ): ''' Helper with dual access, tries set information in RSS and in CS. example: >>> getStorageElementStatus( 'CERN-USER', 'Read' ) S_OK( { 'Read': 'Active' } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( {'Read': 'Active', 'Write': 'Active', 'Check': 'Banned', 'Remove': 'Banned'} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) ''' if self.__getMode(): return self.__setRSSStorageElementStatus( elementName, statusType, status, reason, tokenOwner ) else: return self.__setCSStorageElementStatus( elementName, statusType, status ) ################################################################################ def __updateSECache( self ): ''' Method used to update the StorageElementCache. ''' if not self.__getMode(): # We are using the CS, we do not care about the cache. return { 'OK' : False, 'Message' : 'RSS flag is inactive' } meta = { 'columns' : [ 'Name', 'StatusType', 'Status' ] } rawCache = self.rssClient.selectStatusElement( 'Resource', 'Status', elementType = 'StorageElement', meta = meta ) #This returns S_OK( [['StatusType1','Status1'],['StatusType2','Status2']...] if not rawCache[ 'OK' ]: return rawCache return S_OK( getCacheDictFromList( rawCache[ 'Value' ] ) ) def __cacheMatch( self, resourceNames, statusTypes ): ''' Method that given a resourceName and a statusType, gives the match with the cache. Both arguments can be None, String of list( String, ). Being string, if not present in the cache, there is no match. Keys in the cache are stored as: ( <resourceName>, <statusType> ) ( <resourceName>, <statusType> ) ( <resourceName>, <statusType> ) so, we first need some processing to see which of all possible combinations of resourceName and statusType are in the cache. ( If any of them is None, it is interpreted as all ). ''' cacheKeys = self.seCache.getCacheKeys() if not cacheKeys[ 'OK' ]: return cacheKeys cacheKeys = cacheKeys[ 'Value' ] elementCandidates = cacheKeys if resourceNames is not None: elementCandidates = [] if isinstance( resourceNames, str ): resourceNames = [ resourceNames ] for resourceName in resourceNames: found = False for cK in cacheKeys: if cK[ 0 ] == resourceName: elementCandidates.append( cK ) found = True if not found: return S_ERROR( '%s not found in the cache' % resourceName ) statusTypeCandidates = elementCandidates # now we loop over elementCandidates, saves lots of iterations. if statusTypes is not None: statusTypeCandidates = [] if isinstance( statusTypes, str ): statusTypes = [ statusTypes ] for elementCandidate in elementCandidates: for statusType in statusTypes: if elementCandidate[ 1 ] == statusType: statusTypeCandidates.append( elementCandidate ) return S_OK( statusTypeCandidates ) def __getFromCache( self, elementName, statusType ): ''' Given an elementName and a statusType, matches the cache, and in case of positive match, formats the output and returns ''' match = self.__cacheMatch( elementName, statusType ) if not match[ 'OK' ]: return match cacheMatches = self.seCache.getBulk( match[ 'Value' ] ) if not cacheMatches[ 'OK' ]: return cacheMatches cacheMatches = cacheMatches[ 'Value' ] if not cacheMatches: return S_ERROR( 'Empty cache for ( %s, %s )' % ( elementName, statusType ) ) # We undo the key into <resourceName> and <statusType> fromList = [ list( key ) + [ value ] for key, value in cacheMatches.items() ] return S_OK( getDictFromList( fromList ) ) ################################################################################ def __getRSSStorageElementStatus( self, elementName, statusType, default ): ''' Gets from the cache or the RSS the StorageElements status ''' #Checks cache first cache = self.__getFromCache( elementName, statusType ) if cache[ 'OK' ]: return cache #Humm, seems cache did not work gLogger.info( 'Cache miss with %s %s' % ( elementName, statusType ) ) meta = { 'columns' : [ 'Name', 'StatusType', 'Status' ] } #This returns S_OK( [['StatusType1','Status1'],['StatusType2','Status2']...] res = self.rssClient.selectStatusElement( 'Resource', 'Status', elementType = 'StorageElement', name = elementName, statusType = statusType, meta = meta ) if res[ 'OK' ] and res[ 'Value' ]: return S_OK( getDictFromList( res[ 'Value' ] ) ) if not isinstance( elementName, list ): elementName = [ elementName ] if default is not None: # sec check if statusType is None: statusType = '' defList = [ [ el, statusType, default ] for el in elementName ] return S_OK( getDictFromList( defList ) ) if elementName == [ None ]: elementName = [ '' ] _msg = "StorageElement '%s', with statusType '%s' is unknown for RSS." return S_ERROR( _msg % ( ','.join( elementName ), statusType ) ) def __getCSStorageElementStatus( self, elementName, statusType, default ): ''' Gets from the CS the StorageElements status ''' cs_path = "/Resources/StorageElements" if not isinstance( elementName, list ): elementName = [ elementName ] statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) #statuses = self.__opHelper.getOptionsDict( 'RSSConfiguration/GeneralConfig/Resources/StorageElement' ) #statuses = gConfig.getOptionsDict( '/Operations/RSSConfiguration/GeneralConfig/Resources/StorageElement' ) result = {} for element in elementName: if statusType is not None: # Added Active by default #res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Allowed' ) res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Active' ) if res[ 'OK' ] and res[ 'Value' ]: result[ element ] = { statusType : res[ 'Value' ] } else: res = gConfig.getOptionsDict( "%s/%s" % ( cs_path, element ) ) if res[ 'OK' ] and res[ 'Value' ]: elementStatuses = {} for elementStatusType, value in res[ 'Value' ].items(): #k = k.replace( 'Access', '' ) if elementStatusType in statuses: elementStatuses[ elementStatusType ] = value # If there is no status defined in the CS, we add by default Read and # Write as Active. if elementStatuses == {}: #elementStatuses = { 'ReadAccess' : 'Allowed', 'WriteAccess' : 'Allowed' } elementStatuses = { 'ReadAccess' : 'Active', 'WriteAccess' : 'Active' } result[ element ] = elementStatuses if result: return S_OK( result ) if default is not None: # sec check if statusType is None: statusType = 'none' defList = [ [ el, statusType, default ] for el in elementName ] return S_OK( getDictFromList( defList ) ) _msg = "StorageElement '%s', with statusType '%s' is unknown for CS." return S_ERROR( _msg % ( elementName, statusType ) ) def __setRSSStorageElementStatus( self, elementName, statusType, status, reason, tokenOwner ): ''' Sets on the RSS the StorageElements status ''' expiration = datetime.datetime.utcnow() + datetime.timedelta( days = 1 ) self.seCache.acquireLock() res = self.rssClient.modifyStatusElement( 'Resource', 'Status', name = elementName, statusType = statusType, status = status, reason = reason, tokenOwner = tokenOwner, tokenExpiration = expiration ) if res[ 'OK' ]: self.seCache.refreshCacheAndHistory() # Looks dirty, but this way we avoid retaining the lock when using gLogger. self.seCache.releaseLock() if not res[ 'OK' ]: _msg = 'Error updating StorageElement (%s,%s,%s)' % ( elementName, statusType, status ) gLogger.warn( 'RSS: %s' % _msg ) return res def __setCSStorageElementStatus( self, elementName, statusType, status ): ''' Sets on the CS the StorageElements status ''' statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) if not statusType in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res def __getMode( self ): ''' Get's flag defined ( or not ) on the RSSConfiguration. If defined as 1, we use RSS, if not, we use CS. ''' res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False
class ResourceStatus( object ): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ __metaclass__ = DIRACSingleton def __init__( self, rssFlag = None ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = ResourceStatusClient() self.rssFlag = rssFlag if rssFlag is None: self.rssFlag = self.__getMode() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache( cacheLifeTime, self.__updateRssCache ) def getElementStatus( self, elementName, elementType, statusType = None, default = None ): """ Helper function, tries to get information from the RSS for the given Element, otherwise, it gets it from the CS. :param elementName: name of the element :type elementName: str :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement) :type statusType: None, str, list :param default: defult value (meaningful only when rss is InActive) :type default: str :return: S_OK/S_ERROR :rtype: dict :Example: >>> getElementStatus('CE42', 'ComputingElement') S_OK( { 'CE42': { 'all': 'Active' } } } ) >>> getElementStatus('SE1', 'StorageElement', 'ReadAccess') S_OK( { 'SE1': { 'ReadAccess': 'Banned' } } } ) >>> getElementStatus('SE1', 'ThisIsAWrongElementType', 'ReadAccess') S_ERROR( xyz.. ) >>> getElementStatus('ThisIsAWrongName', 'StorageElement', 'WriteAccess') S_ERROR( xyz.. ) >>> getElementStatus('A_file_catalog', 'FileCatalog') S_OK( { 'A_file_catalog': { 'all': 'Active' } } } ) >>> getElementStatus('SE1', 'StorageElement', ['ReadAccess', 'WriteAccess']) S_OK( { 'SE1': { 'ReadAccess': 'Banned' , 'WriteAccess': 'Active'} } } ) >>> getElementStatus('SE1', 'StorageElement') S_OK( { 'SE1': { 'ReadAccess': 'Probing' , 'WriteAccess': 'Active', 'CheckAccess': 'Degraded', 'RemoveAccess': 'Banned'} } } ) """ allowedParameters = ["StorageElement", "ComputingElement", "FTS", "Catalog"] if elementType not in allowedParameters: return S_ERROR("%s in not in the list of the allowed parameters: %s" % (elementType, allowedParameters)) # Apply defaults if not statusType: if elementType == "StorageElement": statusType = ['ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess'] elif elementType == "ComputingElement": statusType = ['all'] elif elementType == "FTS": statusType = ['all'] elif elementType == "Catalog": statusType = ['all'] if self.rssFlag: return self.__getRSSElementStatus( elementName, elementType, statusType ) else: return self.__getCSElementStatus( elementName, elementType, statusType, default ) def setElementStatus( self, elementName, elementType, statusType, status, reason = None, tokenOwner = None ): """ Tries set information in RSS and in CS. :param elementName: name of the element :type elementName: str :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement) :type statusType: str :param reason: reason for setting the status :type reason: str :param tokenOwner: owner of the token (meaningful only when rss is Active) :type tokenOwner: str :return: S_OK/S_ERROR :rtype: dict :Example: >>> setElementStatus('CE42', 'ComputingElement', 'all', 'Active') S_OK( xyz.. ) >>> setElementStatus('SE1', 'StorageElement', 'ReadAccess', 'Banned') S_OK( xyz.. ) """ if self.rssFlag: return self.__setRSSElementStatus( elementName, elementType, statusType, status, reason, tokenOwner ) else: return self.__setCSElementStatus( elementName, elementType, statusType, status ) ################################################################################ def __updateRssCache( self ): """ Method used to update the rssCache. It will try 5 times to contact the RSS before giving up """ meta = { 'columns' : [ 'Name', 'ElementType', 'StatusType', 'Status' ] } for ti in range( 5 ): rawCache = self.rssClient.selectStatusElement( 'Resource', 'Status', meta = meta ) if rawCache['OK']: break self.log.warn( "Can't get resource's status", rawCache['Message'] + "; trial %d" % ti ) sleep( math.pow( ti, 2 ) ) self.rssClient = ResourceStatusClient() if not rawCache[ 'OK' ]: return rawCache return S_OK( getCacheDictFromRawData( rawCache[ 'Value' ] ) ) ################################################################################ def __getRSSElementStatus( self, elementName, elementType, statusType ): """ Gets from the cache or the RSS the Elements status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new Element. The period between it is added to the DB and the changes are propagated to the cache will be inconsistent, but not dangerous. Just wait <cacheLifeTime> minutes. :param elementName: name of the element :type elementName: str :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement, otherwise it is 'all' or ['all']) :type statusType: str, list """ cacheMatch = self.rssCache.match( elementName, elementType, statusType ) self.log.debug( '__getRSSElementStatus' ) self.log.debug( cacheMatch ) return cacheMatch def __getCSElementStatus( self, elementName, elementType, statusType, default ): """ Gets from the CS the Element status :param elementName: name of the element :type elementName: str :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement) :type statusType: str, list :param default: defult value :type default: None, str """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just return 'Active' if elementType in ('ComputingElement', 'FTS'): return S_OK( { elementName: { 'all': 'Active'} } ) # If we are here it is because elementType is either 'StorageElement' or 'Catalog' if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" statusType = ['Status'] if not isinstance( elementName, list ): elementName = [ elementName ] if not isinstance( statusType, list ): statusType = [ statusType ] result = {} for element in elementName: for sType in statusType: # Look in standard location, 'Active' by default res = gConfig.getValue( "%s/%s/%s" % ( cs_path, element, sType ), 'Active' ) result.setdefault( element, {} )[sType] = res if result: return S_OK( result ) if default is not None: defList = [ [ el, statusType, default ] for el in elementName ] return S_OK( getDictFromList( defList ) ) _msg = "Element '%s', with statusType '%s' is unknown for CS." return S_ERROR( DErrno.ERESUNK, _msg % ( elementName, statusType ) ) def __setRSSElementStatus( self, elementName, elementType, statusType, status, reason, tokenOwner ): """ Sets on the RSS the Elements status """ expiration = datetime.utcnow() + timedelta( days = 1 ) self.rssCache.acquireLock() try: res = self.rssClient.addOrModifyStatusElement( 'Resource', 'Status', name = elementName, elementType = elementType, status = status, statusType = statusType, reason = reason, tokenOwner = tokenOwner, tokenExpiration = expiration ) if res[ 'OK' ]: self.rssCache.refreshCache() if not res[ 'OK' ]: _msg = 'Error updating Element (%s,%s,%s)' % ( elementName, statusType, status ) gLogger.warn( 'RSS: %s' % _msg ) return res finally: # Release lock, no matter what. self.rssCache.releaseLock() def __setCSElementStatus( self, elementName, elementType, statusType, status ): """ Sets on the CS the Elements status """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing if elementType in ('ComputingElement', 'FTS'): return S_OK() # If we are here it is because elementType is either 'StorageElement' or 'Catalog' statuses = self.rssConfig.getConfigStatusType( elementType ) if statusType not in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" #FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs) # but needs to be VO-aware statusType = 'Status' csAPI = CSAPI() csAPI.setOption( "%s/%s/%s/%s" % ( cs_path, elementName, elementType, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res def __getMode( self ): """ Get's flag defined ( or not ) on the RSSConfiguration. If defined as 1, we use RSS, if not, we use CS. """ res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False def isStorageElementAlwaysBanned( self, seName, statusType ): """ Checks if the AlwaysBanned policy is applied to the SE given as parameter :param seName : string, name of the SE :param statusType : ReadAcces, WriteAccess, RemoveAccess, CheckAccess :returns: S_OK(True/False) """ res = getPoliciesThatApply( {'name' : seName, 'statusType' : statusType} ) if not res['OK']: self.log.error( "isStorageElementAlwaysBanned: unable to get the information", res['Message'] ) return res isAlwaysBanned = 'AlwaysBanned' in [policy['type'] for policy in res['Value']] return S_OK( isAlwaysBanned )
class Synchronizer(object): ''' Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. ''' def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() # this just sets the main owner, "rs_svc" just mean "RSS service" self.tokenOwner = "rs_svc" # if we are running this script as a user (from a CLI), # the username found the proxy will be used as tokenOwner result = getProxyInfo() if result['OK']: self.tokenOwner = result['Value']['username'] def sync(self, _eventName, _params): ''' Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK ''' syncSites = self._syncSites() if not syncSites['OK']: gLogger.error(syncSites['Message']) syncResources = self._syncResources() if not syncResources['OK']: gLogger.error(syncResources['Message']) syncNodes = self._syncNodes() if not syncNodes['OK']: gLogger.error(syncNodes['Message']) return S_OK() def _syncSites(self): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info('-- Synchronizing sites --') # sites in CS res = getSites() if not res['OK']: return res sitesCS = res['Value'] gLogger.verbose('%s sites found in CS' % len(sitesCS)) # sites in RSS result = self.rStatus.selectStatusElement('Site', 'Status', meta={'columns': ['Name']}) if not result['OK']: return result sitesDB = [siteDB[0] for siteDB in result['Value']] # Sites that are in DB but not (anymore) in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose('%s sites to be deleted' % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName) gLogger.verbose('Deleting site %s' % siteName) if not deleteQuery['OK']: return deleteQuery # Sites that are in CS but not (anymore) in DB toBeAdded = list(set(sitesCS).difference(set(sitesDB))) gLogger.verbose('%s site entries to be added' % len(toBeAdded)) for site in toBeAdded: query = self.rStatus.addIfNotThereStatusElement( 'Site', 'Status', name=site, statusType='all', status=self.defaultStatus, elementType='Site', tokenOwner=self.tokenOwner, reason='Synchronized') if not query['OK']: return query return S_OK() def _syncResources(self): ''' Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) ''' gLogger.info('-- Synchronizing Resources --') gLogger.verbose('-> StorageElements') ses = self.__syncStorageElements() if not ses['OK']: gLogger.error(ses['Message']) gLogger.verbose('-> FTS') fts = self.__syncFTS() if not fts['OK']: gLogger.error(fts['Message']) gLogger.verbose('-> FileCatalogs') fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs['OK']: gLogger.error(fileCatalogs['Message']) gLogger.verbose('-> ComputingElements') computingElements = self.__syncComputingElements() if not computingElements['OK']: gLogger.error(computingElements['Message']) gLogger.verbose('-> removing resources that no longer exist in the CS') removingResources = self.__removeNonExistingResourcesFromRM() if not removingResources['OK']: gLogger.error(removingResources['Message']) # FIXME: VOMS return S_OK() def _syncNodes(self): ''' Sync resources: compares CS with DB and does the necessary modifications. ( Queues ) ''' gLogger.info('-- Synchronizing Nodes --') gLogger.verbose('-> Queues') queues = self.__syncQueues() if not queues['OK']: gLogger.error(queues['Message']) return S_OK() def __removeNonExistingResourcesFromRM(self): ''' Remove resources from DowntimeCache table that no longer exist in the CS. ''' if not getServiceURL("ResourceStatus/ResourceManagement"): gLogger.verbose( 'ResourceManagement is not installed, skipping removal of non existing resources...' ) return S_OK() sesHosts = getStorageElementsHosts() if not sesHosts['OK']: return sesHosts sesHosts = sesHosts['Value'] resources = sesHosts ftsServer = getFTS3Servers(hostOnly=True) if ftsServer['OK']: resources.extend(ftsServer['Value']) ce = CSHelpers.getComputingElements() if ce['OK']: resources.extend(ce['Value']) downtimes = self.rManagement.selectDowntimeCache() if not downtimes['OK']: return downtimes # Remove hosts that no longer exist in the CS for host in downtimes['Value']: gLogger.verbose('Checking if %s is still in the CS' % host[0]) if host[0] not in resources: gLogger.verbose('%s is no longer in CS, removing entry...' % host[0]) result = self.rManagement.deleteDowntimeCache(name=host[0]) if not result['OK']: return result return S_OK() def __syncComputingElements(self): ''' Sync ComputingElements: compares CS with DB and does the necessary modifications. ''' cesCS = CSHelpers.getComputingElements() if not cesCS['OK']: return cesCS cesCS = cesCS['Value'] gLogger.verbose('%s Computing elements found in CS' % len(cesCS)) cesDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='ComputingElement', meta={'columns': ['Name']}) if not cesDB['OK']: return cesDB cesDB = [ceDB[0] for ceDB in cesDB['Value']] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(cesDB).difference(set(cesCS))) gLogger.verbose('%s Computing elements to be deleted' % len(toBeDeleted)) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ceName) gLogger.verbose('... %s' % ceName) if not deleteQuery['OK']: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType('ComputingElement') result = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='ComputingElement', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result cesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [(se, statusType) for se in cesCS for statusType in statusTypes] toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple))) gLogger.debug('%s Computing elements entries to be added' % len(toBeAdded)) for ceTuple in toBeAdded: _name = ceTuple[0] _statusType = ceTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'ComputingElement' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncFileCatalogs(self): ''' Sync FileCatalogs: compares CS with DB and does the necessary modifications. ''' catalogsCS = CSHelpers.getFileCatalogs() if not catalogsCS['OK']: return catalogsCS catalogsCS = catalogsCS['Value'] gLogger.verbose('%s File catalogs found in CS' % len(catalogsCS)) catalogsDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='Catalog', meta={'columns': ['Name']}) if not catalogsDB['OK']: return catalogsDB catalogsDB = [catalogDB[0] for catalogDB in catalogsDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(catalogsDB).difference(set(catalogsCS))) gLogger.verbose('%s File catalogs to be deleted' % len(toBeDeleted)) # Delete storage elements for catalogName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', catalogName) gLogger.verbose('... %s' % catalogName) if not deleteQuery['OK']: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType('Catalog') result = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='Catalog', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result sesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. catalogsStatusTuples = [(se, statusType) for se in catalogsCS for statusType in statusTypes] toBeAdded = list(set(catalogsStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s File catalogs entries to be added' % len(toBeAdded)) for catalogTuple in toBeAdded: _name = catalogTuple[0] _statusType = catalogTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'Catalog' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncFTS(self): ''' Sync FTS: compares CS with DB and does the necessary modifications. ''' ftsCS = CSHelpers.getFTS() if not ftsCS['OK']: return ftsCS ftsCS = ftsCS['Value'] gLogger.verbose('%s FTS endpoints found in CS' % len(ftsCS)) ftsDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='FTS', meta={'columns': ['Name']}) if not ftsDB['OK']: return ftsDB ftsDB = [fts[0] for fts in ftsDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(ftsDB).difference(set(ftsCS))) gLogger.verbose('%s FTS endpoints to be deleted' % len(toBeDeleted)) # Delete storage elements for ftsName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ftsName) gLogger.verbose('... %s' % ftsName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('FTS') # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] result = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='FTS', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result sesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. ftsStatusTuples = [(se, statusType) for se in ftsCS for statusType in statusTypes] toBeAdded = list(set(ftsStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s FTS endpoints entries to be added' % len(toBeAdded)) for ftsTuple in toBeAdded: _name = ftsTuple[0] _statusType = ftsTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'FTS' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncStorageElements(self): ''' Sync StorageElements: compares CS with DB and does the necessary modifications. ''' sesCS = DMSHelpers().getStorageElements() gLogger.verbose('%s storage elements found in CS' % len(sesCS)) sesDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='StorageElement', meta={'columns': ['Name']}) if not sesDB['OK']: return sesDB sesDB = [seDB[0] for seDB in sesDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(sesDB).difference(set(sesCS))) gLogger.verbose('%s storage elements to be deleted' % len(toBeDeleted)) # Delete storage elements for sesName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', sesName) gLogger.verbose('... %s' % sesName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('StorageElement') # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] result = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='StorageElement', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result sesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. sesStatusTuples = [(se, statusType) for se in sesCS for statusType in statusTypes] toBeAdded = list(set(sesStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s storage element entries to be added' % len(toBeAdded)) for seTuple in toBeAdded: _name = seTuple[0] _statusType = seTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'StorageElement' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncQueues(self): ''' Sync Queues: compares CS with DB and does the necessary modifications. ''' queuesCS = CSHelpers.getQueuesRSS() if not queuesCS['OK']: return queuesCS queuesCS = queuesCS['Value'] gLogger.verbose('%s Queues found in CS' % len(queuesCS)) queuesDB = self.rStatus.selectStatusElement('Node', 'Status', elementType='Queue', meta={'columns': ['Name']}) if not queuesDB['OK']: return queuesDB queuesDB = [queueDB[0] for queueDB in queuesDB['Value']] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(queuesDB).difference(set(queuesCS))) gLogger.verbose('%s Queues to be deleted' % len(toBeDeleted)) # Delete storage elements for queueName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Node', queueName) gLogger.verbose('... %s' % queueName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('Queue') # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Node' ] result = self.rStatus.selectStatusElement( 'Node', 'Status', elementType='Queue', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result queueTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. queueStatusTuples = [(se, statusType) for se in queuesCS for statusType in statusTypes] toBeAdded = list(set(queueStatusTuples).difference(set(queueTuple))) gLogger.verbose('%s Queue entries to be added' % len(toBeAdded)) for queueTuple in toBeAdded: _name = queueTuple[0] _statusType = queueTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'Queue' query = self.rStatus.addIfNotThereStatusElement( 'Node', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK()
class ResourceStatus(object): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ __metaclass__ = DIRACSingleton def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache('StorageElement', cacheLifeTime, self.__updateSECache) def getStorageElementStatus(self, elementName, statusType=None, default=None): """ Helper with dual access, tries to get information from the RSS for the given StorageElement, otherwise, it gets it from the CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'CERN-USER' : { 'ReadAccess': 'Active' } } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( { 'CERN-USER' : {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'}} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): # We do not apply defaults. If is not on the cache, S_ERROR is returned. return self.__getRSSStorageElementStatus(elementName, statusType) else: return self.__getCSStorageElementStatus(elementName, statusType, default) def setStorageElementStatus(self, elementName, statusType, status, reason=None, tokenOwner=None): """ Helper with dual access, tries set information in RSS and in CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'ReadAccess': 'Active' } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): return self.__setRSSStorageElementStatus(elementName, statusType, status, reason, tokenOwner) else: return self.__setCSStorageElementStatus(elementName, statusType, status) ################################################################################ def __updateSECache(self): """ Method used to update the StorageElementCache. """ meta = {'columns': ['Name', 'StatusType', 'Status']} rawCache = self.rssClient.selectStatusElement( 'Resource', 'Status', elementType='StorageElement', meta=meta) if not rawCache['OK']: return rawCache return S_OK(getCacheDictFromRawData(rawCache['Value'])) ################################################################################ def __getRSSStorageElementStatus(self, elementName, statusType): """ Gets from the cache or the RSS the StorageElements status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new StorageElement. The period between it is added to the DB and the changes are propagated to the cache will be inconsisten, but not dangerous. Just wait <cacheLifeTime> minutes. """ cacheMatch = self.seCache.match(elementName, statusType) self.log.debug('__getRSSStorageElementStatus') self.log.debug(cacheMatch) return cacheMatch def __getCSStorageElementStatus(self, elementName, statusType, default): """ Gets from the CS the StorageElements status """ cs_path = "/Resources/StorageElements" if not isinstance(elementName, list): elementName = [elementName] statuses = self.rssConfig.getConfigStatusType('StorageElement') result = {} for element in elementName: if statusType is not None: # Added Active by default res = gConfig.getOption( "%s/%s/%s" % (cs_path, element, statusType), 'Active') if res['OK'] and res['Value']: result[element] = {statusType: res['Value']} else: res = gConfig.getOptionsDict("%s/%s" % (cs_path, element)) if res['OK'] and res['Value']: elementStatuses = {} for elementStatusType, value in res['Value'].items(): if elementStatusType in statuses: elementStatuses[elementStatusType] = value # If there is no status defined in the CS, we add by default Read and # Write as Active. if elementStatuses == {}: elementStatuses = { 'ReadAccess': 'Active', 'WriteAccess': 'Active' } result[element] = elementStatuses if result: return S_OK(result) if default is not None: # sec check if statusType is None: statusType = 'none' defList = [[el, statusType, default] for el in elementName] return S_OK(getDictFromList(defList)) _msg = "StorageElement '%s', with statusType '%s' is unknown for CS." return S_ERROR(_msg % (elementName, statusType)) def __setRSSStorageElementStatus(self, elementName, statusType, status, reason, tokenOwner): """ Sets on the RSS the StorageElements status """ expiration = datetime.datetime.utcnow() + datetime.timedelta(days=1) self.seCache.acquireLock() try: res = self.rssClient.modifyStatusElement( 'Resource', 'Status', name=elementName, statusType=statusType, status=status, reason=reason, tokenOwner=tokenOwner, tokenExpiration=expiration) if res['OK']: self.seCache.refreshCache() if not res['OK']: _msg = 'Error updating StorageElement (%s,%s,%s)' % ( elementName, statusType, status) gLogger.warn('RSS: %s' % _msg) return res finally: # Release lock, no matter what. self.seCache.releaseLock() def __setCSStorageElementStatus(self, elementName, statusType, status): """ Sets on the CS the StorageElements status """ statuses = self.rssConfig.getConfigStatusType('StorageElement') if not statusType in statuses: gLogger.error("%s is not a valid statusType" % statusType) return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses)) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption("%s/%s/%s" % (cs_path, elementName, statusType), status) res = csAPI.commitChanges() if not res['OK']: gLogger.warn('CS: %s' % res['Message']) return res def __getMode(self): """ Get's flag defined ( or not ) on the RSSConfiguration. If defined as 1, we use RSS, if not, we use CS. """ res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False
class Synchronizer( object ): ''' Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. ''' def __init__( self ): """ Constructor. examples: >>> s = Synchronizer() """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.operations = Operations() self.resources = Resources() self.rStatus = ResourceStatusClient.ResourceStatusClient() self.rssConfig = RssConfiguration() self.diracAdmin = DiracAdmin() def sync( self, _eventName, _params ): ''' Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK ''' defSyncResult = { 'added' : [], 'deleted' : [] } # Sites syncSites = self._syncSites() if not syncSites[ 'OK' ]: self.log.error( syncSites[ 'Message' ] ) syncSites = ( syncSites[ 'OK' ] and syncSites[ 'Value' ] ) or defSyncResult # Resources syncResources = self._syncResources() if not syncResources[ 'OK' ]: self.log.error( syncResources[ 'Message' ] ) syncResources = ( syncResources[ 'OK' ] and syncResources[ 'Value' ] ) or defSyncResult # Nodes syncNodes = self._syncNodes() if not syncNodes[ 'OK' ]: self.log.error( syncNodes[ 'Message' ] ) syncNodes = ( syncNodes[ 'OK' ] and syncNodes[ 'Value' ] ) or defSyncResult # Notify via email to : self.notify( syncSites, syncResources, syncNodes ) return S_OK() def notify( self, syncSites, syncResources, syncNodes ): """ Method sending email notification with the result of the synchronization. Email is sent to Operations( EMail/Production ) email address. examples: >>> s.notify( {}, {}, {} ) >>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} ) >>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, { 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} ) :Parameters: **syncSites** - dict() ( keys: added, deleted ) dictionary with the sites added and deleted from the DB **syncResources** - dict() ( keys: added, deleted ) dictionary with the resources added and deleted from the DB **syncNodes** - dict() ( keys: added, deleted ) dictionary with the nodes added and deleted from the DB :return: S_OK """ # Human readable summary msgBody = self.getBody( syncSites, syncResources, syncNodes ) self.log.info( msgBody ) # Email addresses toAddress = self.operations.getValue( 'EMail/Production', '' ) fromAddress = self.rssConfig.getConfigFromAddress( '' ) if toAddress and fromAddress and msgBody: # Subject of the email setup = gConfig.getValue( 'DIRAC/Setup' ) subject = '[RSS](%s) CS Synchronization' % setup self.diracAdmin.sendMail( toAddress, subject, msgBody, fromAddress = fromAddress ) def getBody( self, syncSites, syncResources, syncNodes ): """ Method that given the outputs of the three synchronization methods builds a human readable string. examples: >>> s.getBody( {}, {}, {} ) '' >>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} ) ''' SITES: Site: deleted:1 RubbishSite ''' >>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, { 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} ) ''' SITES: Site: deleted:1 RubbishSite RESOURCES: Computing: added:2 newCE01 newCE02 ''' :Parameters: **syncSites** - dict() ( keys: added, deleted ) dictionary with the sites added and deleted from the DB **syncResources** - dict() ( keys: added, deleted ) dictionary with the resources added and deleted from the DB **syncNodes** - dict() ( keys: added, deleted ) dictionary with the nodes added and deleted from the DB :return: str """ syncMsg = '' for element, syncResult in [ ( 'SITES', syncSites ), ( 'RESOURCES', syncResources ), ( 'NODES', syncNodes ) ]: elementsMsg = '' for elementType, elements in syncResult.items(): elementMsg = '' if elements[ 'added' ]: elementMsg += '\n %s added: %d \n' % ( elementType, len( elements[ 'added' ] ) ) elementMsg += ' ' + '\n '.join( elements[ 'added' ] ) if elements[ 'deleted' ]: elementMsg += '\n %s deleted: %d \n' % ( elementType, len( elements[ 'deleted' ] ) ) elementMsg += ' ' + '\n '.join( elements[ 'deleted' ] ) if elementMsg: elementsMsg += '\n\n%s:\n' % elementType elementsMsg += elementMsg if elementsMsg: syncMsg += '\n\n%s:' % element + elementsMsg return syncMsg #............................................................................. # Sync methods: Site, Resource & Node def _syncSites( self ): """ Method that synchronizes sites ( using their canonical name: CERN.ch ) with elementType = 'Site'. It gets from the CS the eligible site names and then synchronizes them with the DB. If not on the DB, they are added. If in the DB but not on the CS, they are deleted. examples: >> s._syncSites() S_OK( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] } } ) :return: S_OK( { 'Site' : { 'added' : [], 'deleted' : [] }} ) | S_ERROR """ # Get site names from the CS foundSites = self.resources.getEligibleSites() if not foundSites[ 'OK' ]: return foundSites sites = {} # Synchronize with the DB resSync = self.__dbSync( 'Site', 'Site', foundSites[ 'Value' ] ) if not resSync[ 'OK' ]: self.log.error( 'Error synchronizing Sites' ) self.log.error( resSync[ 'Message' ] ) else: sites = resSync[ 'Value' ] return S_OK( { 'Site' : sites } ) def _syncResources( self ): """ Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary keys. It makes one sync round per key ( elementType ). Gets from the CS the eligible Resource/<elementType> names and then synchronizes them with the DB. If not on the DB, they are added. If in the DB but not on the CS, they are deleted. examples: >>> s._syncResources() S_OK( { 'Computing' : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }, 'Storage' : { 'added' : [], 'deleted' : [] }, ... } ) :return: S_OK( { 'RESOURCE_NODE_MAPPINGKey1' : { 'added' : [], 'deleted' : [] }, ...} ) """ resources = {} # Iterate over the different elementTypes for Resource ( Computing, Storage... ) for elementType in RESOURCE_NODE_MAPPING.keys(): # Get Resource / <elementType> names from CS foundResources = self.resources.getEligibleResources( elementType ) if not foundResources[ 'OK' ]: self.log.error( foundResources[ 'Message' ] ) continue # Translate CS result into a list foundResources = foundResources[ 'Value' ] # Synchronize with the DB resSync = self.__dbSync( 'Resource', elementType, foundResources ) if not resSync[ 'OK' ]: self.log.error( 'Error synchronizing %s %s' % ( 'Resource', elementType ) ) self.log.error( resSync[ 'Message' ] ) else: resources[ elementType ] = resSync[ 'Value' ] return S_OK( resources ) def _syncNodes( self ): """ Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary values. It makes one sync round per key ( elementType ). Gets from the CS the eligible Node/<elementType> names and then synchronizes them with the DB. If not on the DB, they are added. If in the DB but not on the CS, they are deleted. examples: >>> s._syncNodes() S_OK( { 'Queue' : { 'added' : [], 'deleted' : [] }, ... } ) :return: S_OK( { 'RESOURCE_NODE_MAPPINGValue1' : { 'added' : [], 'deleted' : [] }, ...} ) """ nodes = {} # Iterate over the different elementTypes for Node ( Queue, AccessProtocol... ) for elementType in RESOURCE_NODE_MAPPING.values(): # Get Node / <elementType> names from CS foundNodes = self.resources.getEligibleNodes( elementType ) if not foundNodes[ 'OK' ]: self.log.error( foundNodes[ 'Value' ] ) continue # Translate CS result into a list : maps NodeName to SiteName<>NodeName to # avoid duplicates # Looong list comprehension, sorry ! foundNodes = [ '%s<>%s' % ( key, item ) for key, subDict in foundNodes[ 'Value' ].items() for subList in subDict.values() for item in subList ] # Synchronize with the DB resSync = self.__dbSync( 'Node', elementType, foundNodes ) if not resSync[ 'OK' ]: self.log.error( 'Error synchronizing %s %s' % ( 'Node', elementType ) ) self.log.error( resSync[ 'Message' ] ) else: nodes[ elementType ] = resSync[ 'Value' ] return S_OK( nodes ) #............................................................................. # DB sync actions def __dbSync( self, elementFamily, elementType, elementsCS ): """ Method synchronizing CS and DB. Compares <elementsCS> with <elementsDB> given the elementFamily and elementType ( e.g. Resource / Computing ). If there are missing elements in the DB, are inserted. If are missing elements in the CS, are deleted from the DB. Note that the logs from the RSS DB are kept ! ( just in case ). :Parameters: **elementFamily** - str any of the valid element families : Site, Resource, Node **elementType** - str any of the valid element types for <elementFamily> **elementsCS** - list list with the elements for <elementFamily>/<elementType> found in the CS :return: S_OK( { 'added' : [], 'deleted' : [] } ) | S_ERROR """ # deleted, added default response syncRes = { 'deleted' : [], 'added' : [], } # Gets <elementFamily>/<elementType> elements from DB elementsDB = self.rStatus.selectStatusElement( elementFamily, 'Status', elementType = elementType, meta = { 'columns' : [ 'name' ] } ) if not elementsDB[ 'OK' ]: return elementsDB elementsDB = [ elementDB[ 0 ] for elementDB in elementsDB[ 'Value' ] ] # Elements in DB but not in CS -> to be deleted toBeDeleted = list( set( elementsDB ).difference( set( elementsCS ) ) ) if toBeDeleted: resDelete = self.__dbDelete( elementFamily, elementType, toBeDeleted ) if not resDelete[ 'OK' ]: return resDelete else: syncRes[ 'deleted' ] = toBeDeleted # Elements in CS but not in DB -> to be added toBeAdded = list( set( elementsCS ).difference( set( elementsDB ) ) ) if toBeAdded: resInsert = self.__dbInsert( elementFamily, elementType, toBeAdded ) if not resInsert[ 'OK' ]: return resInsert else: syncRes[ 'added' ] = toBeAdded return S_OK( syncRes ) def __dbDelete( self, elementFamily, elementType, toBeDeleted ): """ Method that given the elementFamily and elementType, deletes all entries in the History and Status tables for the given elements in toBeDeleted ( all their status Types ). :Parameters: **elementFamily** - str any of the valid element families : Site, Resource, Node **elementType** - str any of the valid element types for <elementFamily>, just used for logging purposes. **toBeDeleted** - list list with the elements to be deleted :return: S_OK | S_ERROR """ self.log.info( 'Deleting %s %s:' % ( elementFamily, elementType ) ) self.log.info( toBeDeleted ) return self.rStatus._extermineStatusElement( elementFamily, toBeDeleted ) def __dbInsert( self, elementFamily, elementType, toBeAdded ): """ Method that given the elementFamily and elementType, adds all elements in toBeAdded with their respective statusTypes, obtained from the CS. They are synchronized with status 'Unknown' and reason 'Synchronized'. :Parameters: **elementFamily** - str any of the valid element families : Site, Resource, Node **elementType** - str any of the valid element types for <elementFamily> **toBeDeleted** - list list with the elements to be added :return: S_OK | S_ERROR """ self.log.info( 'Adding %s %s:' % ( elementFamily, elementType ) ) self.log.info( toBeAdded ) statusTypes = self.rssConfig.getConfigStatusType( elementType ) for element in toBeAdded: for statusType in statusTypes: resInsert = self.rStatus.addIfNotThereStatusElement( elementFamily, 'Status', name = element, statusType = statusType, status = 'Unknown', elementType = elementType, reason = 'Synchronized') if not resInsert[ 'OK' ]: return resInsert return S_OK() #............................................................................... # # def _syncUsers( self ): # ''' # Sync Users: compares CS with DB and does the necessary modifications. # ''' # # gLogger.verbose( '-- Synchronizing users --') # # usersCS = CSHelpers.getRegistryUsers() # if not usersCS[ 'OK' ]: # return usersCS # usersCS = usersCS[ 'Value' ] # # gLogger.verbose( '%s users found in CS' % len( usersCS ) ) # # usersDB = self.rManagement.selectUserRegistryCache( meta = { 'columns' : [ 'login' ] } ) # if not usersDB[ 'OK' ]: # return usersDB # usersDB = [ userDB[0] for userDB in usersDB[ 'Value' ] ] # # # Users that are in DB but not in CS # toBeDeleted = list( set( usersDB ).difference( set( usersCS.keys() ) ) ) # gLogger.verbose( '%s users to be deleted' % len( toBeDeleted ) ) # # # Delete users # # FIXME: probably it is not needed since there is a DatabaseCleanerAgent # for userLogin in toBeDeleted: # # deleteQuery = self.rManagement.deleteUserRegistryCache( login = userLogin ) # # gLogger.verbose( '... %s' % userLogin ) # if not deleteQuery[ 'OK' ]: # return deleteQuery # # # AddOrModify Users # for userLogin, userDict in usersCS.items(): # # _name = userDict[ 'DN' ].split( '=' )[ -1 ] # _email = userDict[ 'Email' ] # # query = self.rManagement.addOrModifyUserRegistryCache( userLogin, _name, _email ) # gLogger.verbose( '-> %s' % userLogin ) # if not query[ 'OK' ]: # return query # # return S_OK() ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class SiteStatus(object): """ RSS helper to interact with the 'Site' family on the DB. It provides the most demanded functions and a cache to avoid hitting the server too often. It provides four methods to interact with the site statuses: * getSiteStatuses * isUsableSite * getUsableSites * getSites """ __metaclass__ = DIRACSingleton def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssFlag = ResourceStatus().rssFlag self.rsClient = ResourceStatusClient() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache) def __updateRssCache(self): """ Method used to update the rssCache. It will try 5 times to contact the RSS before giving up """ meta = {'columns': ['Name', 'Status']} for ti in xrange(5): rawCache = self.rsClient.selectStatusElement('Site', 'Status', meta=meta) if rawCache['OK']: break self.log.warn("Can't get resource's status", rawCache['Message'] + "; trial %d" % ti) sleep(math.pow(ti, 2)) self.rsClient = ResourceStatusClient() if not rawCache['OK']: return rawCache return S_OK(getCacheDictFromRawData(rawCache['Value'])) def getSiteStatuses(self, siteNames=None): """ Method that queries the database for status of the sites in a given list. A single string site name may also be provides as "siteNames" If the input is None, it is interpreted as * ( all ). If match is positive, the output looks like: { 'test1.test1.org': 'Active', 'test2.test2.org': 'Banned', } examples >>> siteStatus.getSiteStatuses( ['test1.test1.uk', 'test2.test2.net', 'test3.test3.org'] ) S_OK( { 'test1.test1.org': 'Active', 'test2.test2.net': 'Banned', 'test3.test3.org': 'Active' } ) >>> siteStatus.getSiteStatuses( 'NotExists') S_ERROR( ... )) >>> siteStatus.getSiteStatuses( None ) S_OK( { 'test1.test1.org': 'Active', 'test2.test2.net': 'Banned', }, ... } ) :Parameters: **siteNames** - `list` or `str` name(s) of the sites to be matched :return: S_OK() || S_ERROR() """ if self.rssFlag: return self.__getRSSSiteStatus(siteNames) else: siteStatusDict = {} wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') if siteNames: if isinstance(siteNames, basestring): siteNames = [siteNames] for siteName in siteNames: result = wmsAdmin.getSiteMaskStatus(siteName) if not result['OK']: return result else: siteStatusDict[siteName] = result['Value'] else: result = wmsAdmin.getSiteMaskStatus() if not result['OK']: return result else: siteStatusDict = result['Value'] return S_OK(siteStatusDict) def __getRSSSiteStatus(self, siteName=None): """ Gets from the cache or the RSS the Sites status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new Element. The period between it is added to the DB and the changes are propagated to the cache will be inconsistent, but not dangerous. Just wait <cacheLifeTime> minutes. :param siteName: name of the site :type siteName: str :return: dict """ cacheMatch = self.rssCache.match(siteName, '', '') self.log.debug('__getRSSSiteStatus') self.log.debug(cacheMatch) return cacheMatch def getUsableSites(self, siteNames=None): """ Returns all sites that are usable if their statusType is either Active or Degraded; in a list. examples >>> siteStatus.getUsableSites( ['test1.test1.uk', 'test2.test2.net', 'test3.test3.org'] ) S_OK( ['test1.test1.uk', 'test3.test3.org'] ) >>> siteStatus.getUsableSites( None ) S_OK( ['test1.test1.uk', 'test3.test3.org', 'test4.test4.org', 'test5.test5.org', ...] ) >>> siteStatus.getUsableSites( 'NotExists' ) S_ERROR( ... ) :Parameters: **siteNames** - `List` or `str` name(s) of the sites to be matched :return: S_OK() || S_ERROR() """ siteStatusDictRes = self.getSiteStatuses(siteNames) if not siteStatusDictRes['OK']: return siteStatusDictRes siteStatusList = [x[0] for x in siteStatusDictRes['Value'].iteritems() if x[1] in ['Active', 'Degraded']] return S_OK(siteStatusList) def getSites(self, siteState='Active'): """ By default, it gets the currently active site list examples >>> siteStatus.getSites() S_OK( ['test1.test1.uk', 'test3.test3.org'] ) >>> siteStatus.getSites( 'Active' ) S_OK( ['test1.test1.uk', 'test3.test3.org'] ) >>> siteStatus.getSites( 'Banned' ) S_OK( ['test0.test0.uk', ... ] ) >>> siteStatus.getSites( 'All' ) S_OK( ['test1.test1.uk', 'test3.test3.org', 'test4.test4.org', 'test5.test5.org'...] ) >>> siteStatus.getSites( None ) S_ERROR( ... ) :Parameters: **siteState** - `String` state of the sites to be matched :return: S_OK() || S_ERROR() """ if not siteState: return S_ERROR(DErrno.ERESUNK, 'siteState parameter is empty') siteStatusDictRes = self.getSiteStatuses() if not siteStatusDictRes['OK']: return siteStatusDictRes if siteState.capitalize() == 'All': # if no siteState is set return everything siteList = list(siteStatusDictRes['Value']) else: # fix case sensitive string siteState = siteState.capitalize() allowedStateList = ['Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown'] if siteState not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') siteList = [x[0] for x in siteStatusDictRes['Value'].iteritems() if x[1] == siteState] return S_OK(siteList) def setSiteStatus(self, site, status, comment='No comment'): """ Set the status of a site in the 'SiteStatus' table of RSS examples >>> siteStatus.banSite( 'site1.test.test' ) S_OK() >>> siteStatus.banSite( None ) S_ERROR( ... ) :Parameters: **site** - `String` the site that is going to be banned **comment** - `String` reason for banning :return: S_OK() || S_ERROR() """ if not status: return S_ERROR(DErrno.ERESUNK, 'status parameter is empty') # fix case sensitive string status = status.capitalize() allowedStateList = ['Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown'] if status not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') if self.rssFlag: result = getProxyInfo() if result['OK']: tokenOwner = result['Value']['username'] else: return S_ERROR("Unable to get user proxy info %s " % result['Message']) tokenExpiration = datetime.utcnow() + timedelta(days=1) self.rssCache.acquireLock() try: result = self.rsClient.modifyStatusElement('Site', 'Status', status=status, name=site, tokenExpiration=tokenExpiration, reason=comment, tokenOwner=tokenOwner) if result['OK']: self.rssCache.refreshCache() else: _msg = 'Error updating status of site %s to %s' % (site, status) gLogger.warn('RSS: %s' % _msg) # Release lock, no matter what. finally: self.rssCache.releaseLock() else: if status in ['Active', 'Degraded']: result = RPCClient('WorkloadManagement/WMSAdministrator').allowSite() else: result = RPCClient('WorkloadManagement/WMSAdministrator').banSite() return result
class SiteStatus(object): """ RSS helper to interact with the 'Site' family on the DB. It provides the most demanded functions and a cache to avoid hitting the server too often. It provides four methods to interact with the site statuses: * getSiteStatuses * isUsableSite * getUsableSites * getSites """ __metaclass__ = DIRACSingleton def __init__(self): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssFlag = ResourceStatus().rssFlag self.rsClient = ResourceStatusClient() cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache) def __updateRssCache(self): """ Method used to update the rssCache. It will try 5 times to contact the RSS before giving up """ meta = {'columns': ['Name', 'Status']} for ti in xrange(5): rawCache = self.rsClient.selectStatusElement('Site', 'Status', meta=meta) if rawCache['OK']: break self.log.warn("Can't get resource's status", rawCache['Message'] + "; trial %d" % ti) sleep(math.pow(ti, 2)) self.rsClient = ResourceStatusClient() if not rawCache['OK']: return rawCache return S_OK(getCacheDictFromRawData(rawCache['Value'])) def getSiteStatuses(self, siteNames=None): """ Method that queries the database for status of the sites in a given list. A single string site name may also be provides as "siteNames" If the input is None, it is interpreted as * ( all ). If match is positive, the output looks like: { 'test1.test1.org': 'Active', 'test2.test2.org': 'Banned', } examples >>> siteStatus.getSiteStatuses( ['test1.test1.uk', 'test2.test2.net', 'test3.test3.org'] ) S_OK( { 'test1.test1.org': 'Active', 'test2.test2.net': 'Banned', 'test3.test3.org': 'Active' } ) >>> siteStatus.getSiteStatuses( 'NotExists') S_ERROR( ... )) >>> siteStatus.getSiteStatuses( None ) S_OK( { 'test1.test1.org': 'Active', 'test2.test2.net': 'Banned', }, ... } ) :Parameters: **siteNames** - `list` or `str` name(s) of the sites to be matched :return: S_OK() || S_ERROR() """ if self.rssFlag: return self.__getRSSSiteStatus(siteNames) else: siteStatusDict = {} wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') if siteNames: if isinstance(siteNames, basestring): siteNames = [siteNames] for siteName in siteNames: result = wmsAdmin.getSiteMaskStatus(siteName) if not result['OK']: return result else: siteStatusDict[siteName] = result['Value'] else: result = wmsAdmin.getSiteMaskStatus() if not result['OK']: return result else: siteStatusDict = result['Value'] return S_OK(siteStatusDict) def __getRSSSiteStatus(self, siteName=None): """ Gets from the cache or the RSS the Sites status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new Element. The period between it is added to the DB and the changes are propagated to the cache will be inconsistent, but not dangerous. Just wait <cacheLifeTime> minutes. :param siteName: name of the site :type siteName: str :return: dict """ cacheMatch = self.rssCache.match(siteName, '', '') self.log.debug('__getRSSSiteStatus') self.log.debug(cacheMatch) return cacheMatch def getUsableSites(self, siteNames=None): """ Returns all sites that are usable if their statusType is either Active or Degraded; in a list. examples >>> siteStatus.getUsableSites( ['test1.test1.uk', 'test2.test2.net', 'test3.test3.org'] ) S_OK( ['test1.test1.uk', 'test3.test3.org'] ) >>> siteStatus.getUsableSites( None ) S_OK( ['test1.test1.uk', 'test3.test3.org', 'test4.test4.org', 'test5.test5.org', ...] ) >>> siteStatus.getUsableSites( 'NotExists' ) S_ERROR( ... ) :Parameters: **siteNames** - `List` or `str` name(s) of the sites to be matched :return: S_OK() || S_ERROR() """ siteStatusDictRes = self.getSiteStatuses(siteNames) if not siteStatusDictRes['OK']: return siteStatusDictRes siteStatusList = [ x[0] for x in siteStatusDictRes['Value'].iteritems() if x[1] in ['Active', 'Degraded'] ] return S_OK(siteStatusList) def getSites(self, siteState='Active'): """ By default, it gets the currently active site list examples >>> siteStatus.getSites() S_OK( ['test1.test1.uk', 'test3.test3.org'] ) >>> siteStatus.getSites( 'Active' ) S_OK( ['test1.test1.uk', 'test3.test3.org'] ) >>> siteStatus.getSites( 'Banned' ) S_OK( ['test0.test0.uk', ... ] ) >>> siteStatus.getSites( 'All' ) S_OK( ['test1.test1.uk', 'test3.test3.org', 'test4.test4.org', 'test5.test5.org'...] ) >>> siteStatus.getSites( None ) S_ERROR( ... ) :Parameters: **siteState** - `String` state of the sites to be matched :return: S_OK() || S_ERROR() """ if not siteState: return S_ERROR(DErrno.ERESUNK, 'siteState parameter is empty') siteStatusDictRes = self.getSiteStatuses() if not siteStatusDictRes['OK']: return siteStatusDictRes if siteState.capitalize() == 'All': # if no siteState is set return everything siteList = list(siteStatusDictRes['Value']) else: # fix case sensitive string siteState = siteState.capitalize() allowedStateList = [ 'Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown' ] if siteState not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') siteList = [ x[0] for x in siteStatusDictRes['Value'].iteritems() if x[1] == siteState ] return S_OK(siteList) def setSiteStatus(self, site, status, comment='No comment'): """ Set the status of a site in the 'SiteStatus' table of RSS examples >>> siteStatus.banSite( 'site1.test.test' ) S_OK() >>> siteStatus.banSite( None ) S_ERROR( ... ) :Parameters: **site** - `String` the site that is going to be banned **comment** - `String` reason for banning :return: S_OK() || S_ERROR() """ if not status: return S_ERROR(DErrno.ERESUNK, 'status parameter is empty') # fix case sensitive string status = status.capitalize() allowedStateList = [ 'Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown' ] if status not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') if self.rssFlag: result = getProxyInfo() if result['OK']: tokenOwner = result['Value']['username'] else: return S_ERROR("Unable to get user proxy info %s " % result['Message']) tokenExpiration = datetime.utcnow() + timedelta(days=1) self.rssCache.acquireLock() try: result = self.rsClient.modifyStatusElement( 'Site', 'Status', status=status, name=site, tokenExpiration=tokenExpiration, reason=comment, tokenOwner=tokenOwner) if result['OK']: self.rssCache.refreshCache() else: _msg = 'Error updating status of site %s to %s' % (site, status) gLogger.warn('RSS: %s' % _msg) # Release lock, no matter what. finally: self.rssCache.releaseLock() else: if status in ['Active', 'Degraded']: result = RPCClient( 'WorkloadManagement/WMSAdministrator').allowSite() else: result = RPCClient( 'WorkloadManagement/WMSAdministrator').banSite() return result
class ResourceStatus(object): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ __metaclass__ = DIRACSingleton def __init__(self, rssFlag=None): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = ResourceStatusClient() self.rssFlag = rssFlag if rssFlag is None: self.rssFlag = self.__getMode() cacheLifeTime = int(self.rssConfig.getConfigCache()) # RSSCache only affects the calls directed to RSS, if using the CS it is not used. self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache) def getElementStatus(self, elementName, elementType, statusType=None, default=None): """ Helper function, tries to get information from the RSS for the given Element, otherwise, it gets it from the CS. :param elementName: name of the element or list of element names :type elementName: str, list :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement) :type statusType: None, str, list :param default: defult value (meaningful only when rss is InActive) :type default: str :return: S_OK/S_ERROR :rtype: dict :Example: >>> getElementStatus('CE42', 'ComputingElement') S_OK( { 'CE42': { 'all': 'Active' } } } ) >>> getElementStatus('SE1', 'StorageElement', 'ReadAccess') S_OK( { 'SE1': { 'ReadAccess': 'Banned' } } } ) >>> getElementStatus('SE1', 'ThisIsAWrongElementType', 'ReadAccess') S_ERROR( xyz.. ) >>> getElementStatus('ThisIsAWrongName', 'StorageElement', 'WriteAccess') S_ERROR( xyz.. ) >>> getElementStatus('A_file_catalog', 'FileCatalog') S_OK( { 'A_file_catalog': { 'all': 'Active' } } } ) >>> getElementStatus('SE1', 'StorageElement', ['ReadAccess', 'WriteAccess']) S_OK( { 'SE1': { 'ReadAccess': 'Banned' , 'WriteAccess': 'Active'} } } ) >>> getElementStatus('SE1', 'StorageElement') S_OK( { 'SE1': { 'ReadAccess': 'Probing' , 'WriteAccess': 'Active', 'CheckAccess': 'Degraded', 'RemoveAccess': 'Banned'} } } ) >>> getElementStatus(['CE1', 'CE2'], 'ComputingElement') S_OK( {'CE1': {'all': 'Active'}, 'CE2': {'all': 'Probing'}}} """ allowedParameters = ["StorageElement", "ComputingElement", "FTS", "Catalog"] if elementType not in allowedParameters: return S_ERROR("%s in not in the list of the allowed parameters: %s" % (elementType, allowedParameters)) # Apply defaults if not statusType: if elementType == "StorageElement": statusType = ['ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess'] elif elementType == "ComputingElement": statusType = ['all'] elif elementType == "FTS": statusType = ['all'] elif elementType == "Catalog": statusType = ['all'] if self.rssFlag: return self.__getRSSElementStatus(elementName, elementType, statusType) else: return self.__getCSElementStatus(elementName, elementType, statusType, default) def setElementStatus(self, elementName, elementType, statusType, status, reason=None, tokenOwner=None): """ Tries set information in RSS and in CS. :param elementName: name of the element :type elementName: str :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement) :type statusType: str :param reason: reason for setting the status :type reason: str :param tokenOwner: owner of the token (meaningful only when rss is Active) :type tokenOwner: str :return: S_OK/S_ERROR :rtype: dict :Example: >>> setElementStatus('CE42', 'ComputingElement', 'all', 'Active') S_OK( xyz.. ) >>> setElementStatus('SE1', 'StorageElement', 'ReadAccess', 'Banned') S_OK( xyz.. ) """ if self.rssFlag: return self.__setRSSElementStatus(elementName, elementType, statusType, status, reason, tokenOwner) else: return self.__setCSElementStatus(elementName, elementType, statusType, status) ################################################################################ def __updateRssCache(self): """ Method used to update the rssCache. It will try 5 times to contact the RSS before giving up """ meta = {'columns': ['Name', 'ElementType', 'StatusType', 'Status']} for ti in range(5): rawCache = self.rssClient.selectStatusElement('Resource', 'Status', meta=meta) if rawCache['OK']: break self.log.warn("Can't get resource's status", rawCache['Message'] + "; trial %d" % ti) sleep(math.pow(ti, 2)) self.rssClient = ResourceStatusClient() if not rawCache['OK']: return rawCache return S_OK(getCacheDictFromRawData(rawCache['Value'])) ################################################################################ def __getRSSElementStatus(self, elementName, elementType, statusType): """ Gets from the cache or the RSS the Elements status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new Element. The period between it is added to the DB and the changes are propagated to the cache will be inconsistent, but not dangerous. Just wait <cacheLifeTime> minutes. :param elementName: name of the element or list of element names :type elementName: str, list :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement, otherwise it is 'all' or ['all']) :type statusType: str, list """ cacheMatch = self.rssCache.match(elementName, elementType, statusType) self.log.debug('__getRSSElementStatus') self.log.debug(cacheMatch) return cacheMatch def __getCSElementStatus(self, elementName, elementType, statusType, default): """ Gets from the CS the Element status :param elementName: name of the element :type elementName: str :param elementType: type of the element (StorageElement, ComputingElement, FTS, Catalog) :type elementType: str :param statusType: type of the status (meaningful only when elementType==StorageElement) :type statusType: str, list :param default: defult value :type default: None, str """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just return 'Active' if elementType in ('ComputingElement', 'FTS'): return S_OK({elementName: {'all': 'Active'}}) # If we are here it is because elementType is either 'StorageElement' or 'Catalog' if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" statusType = ['Status'] if not isinstance(elementName, list): elementName = [elementName] if not isinstance(statusType, list): statusType = [statusType] result = {} for element in elementName: for sType in statusType: # Look in standard location, 'Active' by default res = gConfig.getValue("%s/%s/%s" % (cs_path, element, sType), 'Active') result.setdefault(element, {})[sType] = res if result: return S_OK(result) if default is not None: defList = [[el, statusType, default] for el in elementName] return S_OK(getDictFromList(defList)) _msg = "Element '%s', with statusType '%s' is unknown for CS." return S_ERROR(DErrno.ERESUNK, _msg % (elementName, statusType)) def __setRSSElementStatus(self, elementName, elementType, statusType, status, reason, tokenOwner): """ Sets on the RSS the Elements status """ expiration = datetime.utcnow() + timedelta(days=1) self.rssCache.acquireLock() try: res = self.rssClient.addOrModifyStatusElement('Resource', 'Status', name=elementName, elementType=elementType, status=status, statusType=statusType, reason=reason, tokenOwner=tokenOwner, tokenExpiration=expiration) if res['OK']: self.rssCache.refreshCache() if not res['OK']: _msg = 'Error updating Element (%s,%s,%s)' % (elementName, statusType, status) gLogger.warn('RSS: %s' % _msg) return res finally: # Release lock, no matter what. self.rssCache.releaseLock() def __setCSElementStatus(self, elementName, elementType, statusType, status): """ Sets on the CS the Elements status """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing if elementType in ('ComputingElement', 'FTS'): return S_OK() # If we are here it is because elementType is either 'StorageElement' or 'Catalog' statuses = self.rssConfig.getConfigStatusType(elementType) if statusType not in statuses: gLogger.error("%s is not a valid statusType" % statusType) return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses)) if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" # FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs) # but needs to be VO-aware statusType = 'Status' csAPI = CSAPI() csAPI.setOption("%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType), status) res = csAPI.commitChanges() if not res['OK']: gLogger.warn('CS: %s' % res['Message']) return res def __getMode(self): """ Gets flag defined (or not) on the RSSConfiguration. If defined as 'Active', we use RSS, if not, we use the CS when possible (and WMS for Sites). """ res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False def isStorageElementAlwaysBanned(self, seName, statusType): """ Checks if the AlwaysBanned policy is applied to the SE given as parameter :param seName: string, name of the SE :param statusType: ReadAcces, WriteAccess, RemoveAccess, CheckAccess :returns: S_OK(True/False) """ res = getPoliciesThatApply({'name': seName, 'statusType': statusType}) if not res['OK']: self.log.error("isStorageElementAlwaysBanned: unable to get the information", res['Message']) return res isAlwaysBanned = 'AlwaysBanned' in [policy['type'] for policy in res['Value']] return S_OK(isAlwaysBanned)
class Synchronizer( object ): ''' Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. ''' def __init__( self, rStatus = None, rManagement = None ): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient.ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.rssConfig = RssConfiguration() def sync( self, _eventName, _params ): ''' Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK ''' syncSites = self._syncSites() if not syncSites[ 'OK' ]: gLogger.error( syncSites[ 'Message' ] ) syncResources = self._syncResources() if not syncResources[ 'OK' ]: gLogger.error( syncResources[ 'Message' ] ) syncNodes = self._syncNodes() if not syncNodes[ 'OK' ]: gLogger.error( syncNodes[ 'Message' ] ) #FIXME: also sync users return S_OK() ## Protected methods ######################################################### def _syncSites( self ): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info( '-- Synchronizing sites --') domainSitesCS = CSHelpers.getDomainSites() if not domainSitesCS[ 'OK' ]: return domainSitesCS domainSitesCS = domainSitesCS[ 'Value' ] for domainName, sitesCS in domainSitesCS.items(): gLogger.verbose( '%s sites found in CS for %s domain' % ( len( sitesCS ), domainName ) ) sitesDB = self.rStatus.selectStatusElement( 'Site', 'Status', elementType = domainName, meta = { 'columns' : [ 'name' ] } ) if not sitesDB[ 'OK' ]: return sitesDB sitesDB = [ siteDB[0] for siteDB in sitesDB[ 'Value' ] ] # Sites that are in DB but not in CS toBeDeleted = list( set( sitesDB ).difference( set( sitesCS ) ) ) gLogger.verbose( '%s sites to be deleted' % len( toBeDeleted ) ) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName ) gLogger.verbose( '... %s' % siteName ) if not deleteQuery[ 'OK' ]: return deleteQuery sitesTuple = self.rStatus.selectStatusElement( 'Site', 'Status', elementType = domainName, meta = { 'columns' : [ 'name', 'statusType' ] } ) if not sitesTuple[ 'OK' ]: return sitesTuple sitesTuple = sitesTuple[ 'Value' ] statusTypes = self.rssConfig.getConfigStatusType( domainName ) # For each ( site, statusType ) tuple not present in the DB, add it. siteStatusTuples = [ ( site, statusType ) for site in sitesCS for statusType in statusTypes ] toBeAdded = list( set( siteStatusTuples ).difference( set( sitesTuple ) ) ) gLogger.verbose( '%s site entries to be added' % len( toBeAdded ) ) for siteTuple in toBeAdded: query = self.rStatus.addIfNotThereStatusElement( 'Site', 'Status', name = siteTuple[ 0 ], statusType = siteTuple[ 1 ], status = 'Unknown', elementType = domainName, reason = 'Synchronized' ) if not query[ 'OK' ]: return query return S_OK() def _syncResources( self ): ''' Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) ''' gLogger.info( '-- Synchronizing Resources --' ) gLogger.verbose( '-> StorageElements' ) ses = self.__syncStorageElements() if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) gLogger.verbose( '-> FTS' ) fts = self.__syncFTS() if not fts[ 'OK' ]: gLogger.error( fts[ 'Message' ] ) gLogger.verbose( '-> FileCatalogs' ) fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs[ 'OK' ]: gLogger.error( fileCatalogs[ 'Message' ] ) gLogger.verbose( '-> ComputingElements' ) computingElements = self.__syncComputingElements() if not computingElements[ 'OK' ]: gLogger.error( computingElements[ 'Message' ] ) #FIXME: VOMS return S_OK() def _syncNodes( self ): ''' Sync resources: compares CS with DB and does the necessary modifications. ( Queues ) ''' gLogger.info( '-- Synchronizing Nodes --' ) gLogger.verbose( '-> Queues' ) queues = self.__syncQueues() if not queues[ 'OK' ]: gLogger.error( queues[ 'Message' ] ) return S_OK() ## Private methods ########################################################### def __syncComputingElements( self ): ''' Sync CEs: compares CS with DB and does the necessary modifications. ''' cesCS = CSHelpers.getComputingElements() if not cesCS[ 'OK' ]: return cesCS cesCS = cesCS[ 'Value' ] gLogger.verbose( '%s Computing elements found in CS' % len( cesCS ) ) cesDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'CE', meta = { 'columns' : [ 'name' ] } ) if not cesDB[ 'OK' ]: return cesDB cesDB = [ ceDB[0] for ceDB in cesDB[ 'Value' ] ] # ComputingElements that are in DB but not in CS toBeDeleted = list( set( cesDB ).difference( set( cesCS ) ) ) gLogger.verbose( '%s Computing elements to be deleted' % len( toBeDeleted ) ) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ceName ) gLogger.verbose( '... %s' % ceName ) if not deleteQuery[ 'OK' ]: return deleteQuery #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType( 'CE' ) cesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'CE', meta = { 'columns' : [ 'name', 'statusType' ] } ) if not cesTuple[ 'OK' ]: return cesTuple cesTuple = cesTuple[ 'Value' ] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [ ( se, statusType ) for se in cesCS for statusType in statusTypes ] toBeAdded = list( set( cesStatusTuples ).difference( set( cesTuple ) ) ) gLogger.debug( '%s Computing elements entries to be added' % len( toBeAdded ) ) for ceTuple in toBeAdded: _name = ceTuple[ 0 ] _statusType = ceTuple[ 1 ] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'CE' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name = _name, statusType = _statusType, status = _status, elementType = _elementType, reason = _reason ) if not query[ 'OK' ]: return query return S_OK() def __syncFileCatalogs( self ): ''' Sync FileCatalogs: compares CS with DB and does the necessary modifications. ''' catalogsCS = CSHelpers.getFileCatalogs() if not catalogsCS[ 'OK' ]: return catalogsCS catalogsCS = catalogsCS[ 'Value' ] gLogger.verbose( '%s File catalogs found in CS' % len( catalogsCS ) ) catalogsDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'Catalog', meta = { 'columns' : [ 'name' ] } ) if not catalogsDB[ 'OK' ]: return catalogsDB catalogsDB = [ catalogDB[0] for catalogDB in catalogsDB[ 'Value' ] ] # StorageElements that are in DB but not in CS toBeDeleted = list( set( catalogsDB ).difference( set( catalogsCS ) ) ) gLogger.verbose( '%s File catalogs to be deleted' % len( toBeDeleted ) ) # Delete storage elements for catalogName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', catalogName ) gLogger.verbose( '... %s' % catalogName ) if not deleteQuery[ 'OK' ]: return deleteQuery #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType( 'Catalog' ) sesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'Catalog', meta = { 'columns' : [ 'name', 'statusType' ] } ) if not sesTuple[ 'OK' ]: return sesTuple sesTuple = sesTuple[ 'Value' ] # For each ( se, statusType ) tuple not present in the DB, add it. catalogsStatusTuples = [ ( se, statusType ) for se in catalogsCS for statusType in statusTypes ] toBeAdded = list( set( catalogsStatusTuples ).difference( set( sesTuple ) ) ) gLogger.verbose( '%s File catalogs entries to be added' % len( toBeAdded ) ) for catalogTuple in toBeAdded: _name = catalogTuple[ 0 ] _statusType = catalogTuple[ 1 ] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'Catalog' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name = _name, statusType = _statusType, status = _status, elementType = _elementType, reason = _reason ) if not query[ 'OK' ]: return query return S_OK() def __syncFTS( self ): ''' Sync FTS: compares CS with DB and does the necessary modifications. ''' ftsCS = CSHelpers.getFTS() if not ftsCS[ 'OK' ]: return ftsCS ftsCS = ftsCS[ 'Value' ] gLogger.verbose( '%s FTS endpoints found in CS' % len( ftsCS ) ) ftsDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'FTS', meta = { 'columns' : [ 'name' ] } ) if not ftsDB[ 'OK' ]: return ftsDB ftsDB = [ fts[0] for fts in ftsDB[ 'Value' ] ] # StorageElements that are in DB but not in CS toBeDeleted = list( set( ftsDB ).difference( set( ftsCS ) ) ) gLogger.verbose( '%s FTS endpoints to be deleted' % len( toBeDeleted ) ) # Delete storage elements for ftsName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ftsName ) gLogger.verbose( '... %s' % ftsName ) if not deleteQuery[ 'OK' ]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType( 'FTS' ) #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] sesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'FTS', meta = { 'columns' : [ 'name', 'statusType' ] } ) if not sesTuple[ 'OK' ]: return sesTuple sesTuple = sesTuple[ 'Value' ] # For each ( se, statusType ) tuple not present in the DB, add it. ftsStatusTuples = [ ( se, statusType ) for se in ftsCS for statusType in statusTypes ] toBeAdded = list( set( ftsStatusTuples ).difference( set( sesTuple ) ) ) gLogger.verbose( '%s FTS endpoints entries to be added' % len( toBeAdded ) ) for ftsTuple in toBeAdded: _name = ftsTuple[ 0 ] _statusType = ftsTuple[ 1 ] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'FTS' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name = _name, statusType = _statusType, status = _status, elementType = _elementType, reason = _reason ) if not query[ 'OK' ]: return query return S_OK() def __syncStorageElements( self ): ''' Sync StorageElements: compares CS with DB and does the necessary modifications. ''' sesCS = CSHelpers.getStorageElements() if not sesCS[ 'OK' ]: return sesCS sesCS = sesCS[ 'Value' ] gLogger.verbose( '%s storage elements found in CS' % len( sesCS ) ) sesDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'StorageElement', meta = { 'columns' : [ 'name' ] } ) if not sesDB[ 'OK' ]: return sesDB sesDB = [ seDB[0] for seDB in sesDB[ 'Value' ] ] # StorageElements that are in DB but not in CS toBeDeleted = list( set( sesDB ).difference( set( sesCS ) ) ) gLogger.verbose( '%s storage elements to be deleted' % len( toBeDeleted ) ) # Delete storage elements for sesName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', sesName ) gLogger.verbose( '... %s' % sesName ) if not deleteQuery[ 'OK' ]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType( 'StorageElement' ) #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] sesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType = 'StorageElement', meta = { 'columns' : [ 'name', 'statusType' ] } ) if not sesTuple[ 'OK' ]: return sesTuple sesTuple = sesTuple[ 'Value' ] # For each ( se, statusType ) tuple not present in the DB, add it. sesStatusTuples = [ ( se, statusType ) for se in sesCS for statusType in statusTypes ] toBeAdded = list( set( sesStatusTuples ).difference( set( sesTuple ) ) ) gLogger.verbose( '%s storage element entries to be added' % len( toBeAdded ) ) for seTuple in toBeAdded: _name = seTuple[ 0 ] _statusType = seTuple[ 1 ] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'StorageElement' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name = _name, statusType = _statusType, status = _status, elementType = _elementType, reason = _reason ) if not query[ 'OK' ]: return query return S_OK() def __syncQueues( self ): ''' Sync Queues: compares CS with DB and does the necessary modifications. ''' queuesCS = CSHelpers.getQueues() if not queuesCS[ 'OK' ]: return queuesCS queuesCS = queuesCS[ 'Value' ] gLogger.verbose( '%s Queues found in CS' % len( queuesCS ) ) queuesDB = self.rStatus.selectStatusElement( 'Node', 'Status', elementType = 'Queue', meta = { 'columns' : [ 'name' ] } ) if not queuesDB[ 'OK' ]: return queuesDB queuesDB = [ queueDB[0] for queueDB in queuesDB[ 'Value' ] ] # ComputingElements that are in DB but not in CS toBeDeleted = list( set( queuesDB ).difference( set( queuesCS ) ) ) gLogger.verbose( '%s Queues to be deleted' % len( toBeDeleted ) ) # Delete storage elements for queueName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Node', queueName ) gLogger.verbose( '... %s' % queueName ) if not deleteQuery[ 'OK' ]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType( 'Queue' ) #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Node' ] queueTuple = self.rStatus.selectStatusElement( 'Node', 'Status', elementType = 'Queue', meta = { 'columns' : [ 'name', 'statusType' ] } ) if not queueTuple[ 'OK' ]: return queueTuple queueTuple = queueTuple[ 'Value' ] # For each ( se, statusType ) tuple not present in the DB, add it. queueStatusTuples = [ ( se, statusType ) for se in queuesCS for statusType in statusTypes ] toBeAdded = list( set( queueStatusTuples ).difference( set( queueTuple ) ) ) gLogger.verbose( '%s Queue entries to be added' % len( toBeAdded ) ) for queueTuple in toBeAdded: _name = queueTuple[ 0 ] _statusType = queueTuple[ 1 ] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'Queue' query = self.rStatus.addIfNotThereStatusElement( 'Node', 'Status', name = _name, statusType = _statusType, status = _status, elementType = _elementType, reason = _reason ) if not query[ 'OK' ]: return query return S_OK() def _syncUsers( self ): ''' Sync Users: compares CS with DB and does the necessary modifications. ''' gLogger.verbose( '-- Synchronizing users --') usersCS = CSHelpers.getRegistryUsers() if not usersCS[ 'OK' ]: return usersCS usersCS = usersCS[ 'Value' ] gLogger.verbose( '%s users found in CS' % len( usersCS ) ) usersDB = self.rManagement.selectUserRegistryCache( meta = { 'columns' : [ 'login' ] } ) if not usersDB[ 'OK' ]: return usersDB usersDB = [ userDB[0] for userDB in usersDB[ 'Value' ] ] # Users that are in DB but not in CS toBeDeleted = list( set( usersDB ).difference( set( usersCS.keys() ) ) ) gLogger.verbose( '%s users to be deleted' % len( toBeDeleted ) ) # Delete users # FIXME: probably it is not needed since there is a DatabaseCleanerAgent for userLogin in toBeDeleted: deleteQuery = self.rManagement.deleteUserRegistryCache( login = userLogin ) gLogger.verbose( '... %s' % userLogin ) if not deleteQuery[ 'OK' ]: return deleteQuery # AddOrModify Users for userLogin, userDict in usersCS.items(): _name = userDict[ 'DN' ].split( '=' )[ -1 ] _email = userDict[ 'Email' ] query = self.rManagement.addOrModifyUserRegistryCache( userLogin, _name, _email ) gLogger.verbose( '-> %s' % userLogin ) if not query[ 'OK' ]: return query return S_OK()
class ResourceStatus( object ): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ __metaclass__ = DIRACSingleton def __init__( self ): """ Constructor, initializes the rssClient. """ self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.rssConfig = RssConfiguration() self.__opHelper = Operations() self.rssClient = None self.infoGetter = InfoGetter() # We can set CacheLifetime and CacheHistory from CS, so that we can tune them. cacheLifeTime = int( self.rssConfig.getConfigCache() ) # RSSCache only affects the calls directed to RSS, if using the CS it is not # used. self.seCache = RSSCache( 'StorageElement', cacheLifeTime, self.__updateSECache ) def getStorageElementStatus( self, elementName, statusType = None, default = None ): """ Helper with dual access, tries to get information from the RSS for the given StorageElement, otherwise, it gets it from the CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'CERN-USER' : { 'ReadAccess': 'Active' } } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( { 'CERN-USER' : {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'}} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): # We do not apply defaults. If is not on the cache, S_ERROR is returned. return self.__getRSSStorageElementStatus( elementName, statusType ) else: return self.__getCSStorageElementStatus( elementName, statusType, default ) def setStorageElementStatus( self, elementName, statusType, status, reason = None, tokenOwner = None ): """ Helper with dual access, tries set information in RSS and in CS. example: >>> getStorageElementStatus( 'CERN-USER', 'ReadAccess' ) S_OK( { 'ReadAccess': 'Active' } ) >>> getStorageElementStatus( 'CERN-USER', 'Write' ) S_OK( {'ReadAccess': 'Active', 'WriteAccess': 'Active', 'CheckAccess': 'Banned', 'RemoveAccess': 'Banned'} ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType' ) S_ERROR( xyz.. ) >>> getStorageElementStatus( 'CERN-USER', 'ThisIsAWrongStatusType', 'Unknown' ) S_OK( 'Unknown' ) """ if self.__getMode(): return self.__setRSSStorageElementStatus( elementName, statusType, status, reason, tokenOwner ) else: return self.__setCSStorageElementStatus( elementName, statusType, status ) ################################################################################ def __updateSECache( self ): """ Method used to update the StorageElementCache. It will try 5 times to contact the RSS before giving up """ meta = { 'columns' : [ 'Name', 'StatusType', 'Status' ] } for ti in range( 5 ): rawCache = self.rssClient.selectStatusElement( 'Resource', 'Status', elementType = 'StorageElement', meta = meta ) if rawCache['OK']: break self.log.warn( "Can't get SE status", rawCache['Message'] + "; trial %d" % ti ) sleep( math.pow( ti, 2 ) ) self.rssClient = ResourceStatusClient() if not rawCache[ 'OK' ]: return rawCache return S_OK( getCacheDictFromRawData( rawCache[ 'Value' ] ) ) ################################################################################ def __getRSSStorageElementStatus( self, elementName, statusType ): """ Gets from the cache or the RSS the StorageElements status. The cache is a copy of the DB table. If it is not on the cache, most likely is not going to be on the DB. There is one exception: item just added to the CS, e.g. new StorageElement. The period between it is added to the DB and the changes are propagated to the cache will be inconsisten, but not dangerous. Just wait <cacheLifeTime> minutes. """ cacheMatch = self.seCache.match( elementName, statusType ) self.log.debug( '__getRSSStorageElementStatus' ) self.log.debug( cacheMatch ) return cacheMatch def __getCSStorageElementStatus( self, elementName, statusType, default ): """ Gets from the CS the StorageElements status """ cs_path = "/Resources/StorageElements" if not isinstance( elementName, list ): elementName = [ elementName ] statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) result = {} for element in elementName: if statusType is not None: # Added Active by default res = gConfig.getValue( "%s/%s/%s" % ( cs_path, element, statusType ), 'Active' ) result[element] = {statusType: res} else: res = gConfig.getOptionsDict( "%s/%s" % ( cs_path, element ) ) if res[ 'OK' ] and res[ 'Value' ]: elementStatuses = {} for elementStatusType, value in res[ 'Value' ].items(): if elementStatusType in statuses: elementStatuses[ elementStatusType ] = value # If there is no status defined in the CS, we add by default Read and # Write as Active. if elementStatuses == {}: elementStatuses = { 'ReadAccess' : 'Active', 'WriteAccess' : 'Active' } result[ element ] = elementStatuses if result: return S_OK( result ) if default is not None: # sec check if statusType is None: statusType = 'none' defList = [ [ el, statusType, default ] for el in elementName ] return S_OK( getDictFromList( defList ) ) _msg = "StorageElement '%s', with statusType '%s' is unknown for CS." return S_ERROR( _msg % ( elementName, statusType ) ) def __setRSSStorageElementStatus( self, elementName, statusType, status, reason, tokenOwner ): """ Sets on the RSS the StorageElements status """ expiration = datetime.datetime.utcnow() + datetime.timedelta( days = 1 ) self.seCache.acquireLock() try: res = self.rssClient.modifyStatusElement( 'Resource', 'Status', name = elementName, statusType = statusType, status = status, reason = reason, tokenOwner = tokenOwner, tokenExpiration = expiration ) if res[ 'OK' ]: self.seCache.refreshCache() if not res[ 'OK' ]: _msg = 'Error updating StorageElement (%s,%s,%s)' % ( elementName, statusType, status ) gLogger.warn( 'RSS: %s' % _msg ) return res finally: # Release lock, no matter what. self.seCache.releaseLock() def __setCSStorageElementStatus( self, elementName, statusType, status ): """ Sets on the CS the StorageElements status """ statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) if not statusType in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res def __getMode( self ): """ Get's flag defined ( or not ) on the RSSConfiguration. If defined as 1, we use RSS, if not, we use CS. """ res = self.rssConfig.getConfigState() if res == 'Active': if self.rssClient is None: self.rssClient = ResourceStatusClient() return True self.rssClient = None return False def isStorageElementAlwaysBanned( self, seName, statusType ): """ Checks if the AlwaysBanned policy is applied to the SE given as parameter :param seName : string, name of the SE :param statusType : ReadAcces, WriteAccess, RemoveAccess, CheckAccess :returns S_OK(True/False) """ res = self.infoGetter.getPoliciesThatApply( {'name' : seName, 'statusType' : statusType} ) if not res['OK']: self.log.error( "isStorageElementAlwaysBanned: unable to get the information", res['Message'] ) return res isAlwaysBanned = 'AlwaysBanned' in [policy['type'] for policy in res['Value']] return S_OK( isAlwaysBanned )
class Synchronizer(object): ''' Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. ''' def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() self.tokenOwner = "rs_svc" result = getProxyInfo() if result['OK']: self.tokenOwner = result['Value']['username'] def sync(self, _eventName, _params): ''' Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK ''' syncSites = self._syncSites() if not syncSites['OK']: gLogger.error(syncSites['Message']) syncResources = self._syncResources() if not syncResources['OK']: gLogger.error(syncResources['Message']) syncNodes = self._syncNodes() if not syncNodes['OK']: gLogger.error(syncNodes['Message']) return S_OK() ## Protected methods ######################################################### def _syncSites(self): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info('-- Synchronizing sites --') # sites in CS res = CSHelpers.getSites() if not res['OK']: return res sitesCS = res['Value'] gLogger.verbose('%s sites found in CS' % len(sitesCS)) # sites in RSS result = self.rStatus.selectStatusElement('Site', 'Status', meta={'columns': ['Name']}) if not result['OK']: return result sitesDB = [siteDB[0] for siteDB in result['Value']] # Sites that are in DB but not (anymore) in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose('%s sites to be deleted' % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName) gLogger.verbose('Deleting site %s' % siteName) if not deleteQuery['OK']: return deleteQuery # Sites that are in CS but not (anymore) in DB toBeAdded = list(set(sitesCS).difference(set(sitesDB))) gLogger.verbose('%s site entries to be added' % len(toBeAdded)) for site in toBeAdded: query = self.rStatus.addIfNotThereStatusElement('Site', 'Status', name=site, statusType='all', status=self.defaultStatus, elementType='Site', tokenOwner=self.tokenOwner, reason='Synchronized') if not query['OK']: return query return S_OK() def _syncResources(self): ''' Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) ''' gLogger.info('-- Synchronizing Resources --') gLogger.verbose('-> StorageElements') ses = self.__syncStorageElements() if not ses['OK']: gLogger.error(ses['Message']) gLogger.verbose('-> FTS') fts = self.__syncFTS() if not fts['OK']: gLogger.error(fts['Message']) gLogger.verbose('-> FileCatalogs') fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs['OK']: gLogger.error(fileCatalogs['Message']) gLogger.verbose('-> ComputingElements') computingElements = self.__syncComputingElements() if not computingElements['OK']: gLogger.error(computingElements['Message']) gLogger.verbose('-> removing resources that no longer exist in the CS') removingResources = self.__removeNonExistingResourcesFromRM() if not removingResources['OK']: gLogger.error(removingResources['Message']) # FIXME: VOMS return S_OK() def _syncNodes(self): ''' Sync resources: compares CS with DB and does the necessary modifications. ( Queues ) ''' gLogger.info('-- Synchronizing Nodes --') gLogger.verbose('-> Queues') queues = self.__syncQueues() if not queues['OK']: gLogger.error(queues['Message']) return S_OK() ## Private methods ########################################################### def __removeNonExistingResourcesFromRM(self): ''' Remove resources from DowntimeCache table that no longer exist in the CS. ''' if not getServiceURL("ResourceStatus/ResourceManagement"): gLogger.verbose( 'ResourceManagement is not installed, skipping removal of non existing resources...') return S_OK() sesHosts = CSHelpers.getStorageElementsHosts() if not sesHosts['OK']: return sesHosts sesHosts = sesHosts['Value'] resources = sesHosts ftsServer = getFTS3Servers() if ftsServer['OK']: resources.extend(ftsServer['Value']) ce = CSHelpers.getComputingElements() if ce['OK']: resources.extend(ce['Value']) downtimes = self.rManagement.selectDowntimeCache() if not downtimes['OK']: return downtimes # Remove hosts that no longer exist in the CS for host in downtimes['Value']: gLogger.verbose('Checking if %s is still in the CS' % host[0]) if host[0] not in resources: gLogger.verbose( '%s is no longer in CS, removing entry...' % host[0]) result = self.rManagement.deleteDowntimeCache(name=host[0]) if not result['OK']: return result return S_OK() def __syncComputingElements(self): ''' Sync ComputingElements: compares CS with DB and does the necessary modifications. ''' cesCS = CSHelpers.getComputingElements() if not cesCS['OK']: return cesCS cesCS = cesCS['Value'] gLogger.verbose('%s Computing elements found in CS' % len(cesCS)) cesDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='ComputingElement', meta={'columns': ['Name']}) if not cesDB['OK']: return cesDB cesDB = [ceDB[0] for ceDB in cesDB['Value']] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(cesDB).difference(set(cesCS))) gLogger.verbose('%s Computing elements to be deleted' % len(toBeDeleted)) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ceName) gLogger.verbose('... %s' % ceName) if not deleteQuery['OK']: return deleteQuery #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType('ComputingElement') result = self.rStatus.selectStatusElement('Resource', 'Status', elementType='ComputingElement', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result cesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [(se, statusType) for se in cesCS for statusType in statusTypes] toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple))) gLogger.debug('%s Computing elements entries to be added' % len(toBeAdded)) for ceTuple in toBeAdded: _name = ceTuple[0] _statusType = ceTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'ComputingElement' query = self.rStatus.addIfNotThereStatusElement('Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncFileCatalogs(self): ''' Sync FileCatalogs: compares CS with DB and does the necessary modifications. ''' catalogsCS = CSHelpers.getFileCatalogs() if not catalogsCS['OK']: return catalogsCS catalogsCS = catalogsCS['Value'] gLogger.verbose('%s File catalogs found in CS' % len(catalogsCS)) catalogsDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='Catalog', meta={'columns': ['Name']}) if not catalogsDB['OK']: return catalogsDB catalogsDB = [catalogDB[0] for catalogDB in catalogsDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(catalogsDB).difference(set(catalogsCS))) gLogger.verbose('%s File catalogs to be deleted' % len(toBeDeleted)) # Delete storage elements for catalogName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', catalogName) gLogger.verbose('... %s' % catalogName) if not deleteQuery['OK']: return deleteQuery #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType('Catalog') result = self.rStatus.selectStatusElement('Resource', 'Status', elementType='Catalog', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result sesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. catalogsStatusTuples = [(se, statusType) for se in catalogsCS for statusType in statusTypes] toBeAdded = list(set(catalogsStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s File catalogs entries to be added' % len(toBeAdded)) for catalogTuple in toBeAdded: _name = catalogTuple[0] _statusType = catalogTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'Catalog' query = self.rStatus.addIfNotThereStatusElement('Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncFTS(self): ''' Sync FTS: compares CS with DB and does the necessary modifications. ''' ftsCS = CSHelpers.getFTS() if not ftsCS['OK']: return ftsCS ftsCS = ftsCS['Value'] gLogger.verbose('%s FTS endpoints found in CS' % len(ftsCS)) ftsDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='FTS', meta={'columns': ['Name']}) if not ftsDB['OK']: return ftsDB ftsDB = [fts[0] for fts in ftsDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(ftsDB).difference(set(ftsCS))) gLogger.verbose('%s FTS endpoints to be deleted' % len(toBeDeleted)) # Delete storage elements for ftsName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ftsName) gLogger.verbose('... %s' % ftsName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('FTS') #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] result = self.rStatus.selectStatusElement('Resource', 'Status', elementType='FTS', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result sesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. ftsStatusTuples = [(se, statusType) for se in ftsCS for statusType in statusTypes] toBeAdded = list(set(ftsStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s FTS endpoints entries to be added' % len(toBeAdded)) for ftsTuple in toBeAdded: _name = ftsTuple[0] _statusType = ftsTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'FTS' query = self.rStatus.addIfNotThereStatusElement('Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncStorageElements(self): ''' Sync StorageElements: compares CS with DB and does the necessary modifications. ''' sesCS = CSHelpers.getStorageElements() if not sesCS['OK']: return sesCS sesCS = sesCS['Value'] gLogger.verbose('%s storage elements found in CS' % len(sesCS)) sesDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='StorageElement', meta={'columns': ['Name']}) if not sesDB['OK']: return sesDB sesDB = [seDB[0] for seDB in sesDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(sesDB).difference(set(sesCS))) gLogger.verbose('%s storage elements to be deleted' % len(toBeDeleted)) # Delete storage elements for sesName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', sesName) gLogger.verbose('... %s' % sesName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('StorageElement') #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] result = self.rStatus.selectStatusElement('Resource', 'Status', elementType='StorageElement', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result sesTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. sesStatusTuples = [(se, statusType) for se in sesCS for statusType in statusTypes] toBeAdded = list(set(sesStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s storage element entries to be added' % len(toBeAdded)) for seTuple in toBeAdded: _name = seTuple[0] _statusType = seTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'StorageElement' query = self.rStatus.addIfNotThereStatusElement('Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK() def __syncQueues(self): ''' Sync Queues: compares CS with DB and does the necessary modifications. ''' queuesCS = CSHelpers.getQueues() if not queuesCS['OK']: return queuesCS queuesCS = queuesCS['Value'] gLogger.verbose('%s Queues found in CS' % len(queuesCS)) queuesDB = self.rStatus.selectStatusElement('Node', 'Status', elementType='Queue', meta={'columns': ['Name']}) if not queuesDB['OK']: return queuesDB queuesDB = [queueDB[0] for queueDB in queuesDB['Value']] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(queuesDB).difference(set(queuesCS))) gLogger.verbose('%s Queues to be deleted' % len(toBeDeleted)) # Delete storage elements for queueName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Node', queueName) gLogger.verbose('... %s' % queueName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('Queue') #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Node' ] result = self.rStatus.selectStatusElement('Node', 'Status', elementType='Queue', meta={'columns': ['Name', 'StatusType']}) if not result['OK']: return result queueTuple = [(x[0], x[1]) for x in result['Value']] # For each ( se, statusType ) tuple not present in the DB, add it. queueStatusTuples = [(se, statusType) for se in queuesCS for statusType in statusTypes] toBeAdded = list(set(queueStatusTuples).difference(set(queueTuple))) gLogger.verbose('%s Queue entries to be added' % len(toBeAdded)) for queueTuple in toBeAdded: _name = queueTuple[0] _statusType = queueTuple[1] _status = self.defaultStatus _reason = 'Synchronized' _elementType = 'Queue' query = self.rStatus.addIfNotThereStatusElement('Node', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason) if not query['OK']: return query return S_OK()
class Synchronizer: """ Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. """ def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() # this just sets the main owner, "rs_svc" just mean "RSS service" self.tokenOwner = "rs_svc" # if we are running this script as a user (from a CLI), # the username found the proxy will be used as tokenOwner result = getProxyInfo() if result["OK"]: self.tokenOwner = result["Value"]["username"] def sync(self, _eventName, _params): """ Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK """ syncSites = self._syncSites() if not syncSites["OK"]: gLogger.error(syncSites["Message"]) syncResources = self._syncResources() if not syncResources["OK"]: gLogger.error(syncResources["Message"]) syncNodes = self._syncNodes() if not syncNodes["OK"]: gLogger.error(syncNodes["Message"]) return S_OK() def _syncSites(self): """ Sync sites: compares CS with DB and does the necessary modifications. """ gLogger.info("-- Synchronizing sites --") # sites in CS res = getSites() if not res["OK"]: return res sitesCS = res["Value"] gLogger.verbose("%s sites found in CS" % len(sitesCS)) # sites in RSS result = self.rStatus.selectStatusElement("Site", "Status", meta={"columns": ["Name"]}) if not result["OK"]: return result sitesDB = [siteDB[0] for siteDB in result["Value"]] # Sites that are in DB but not (anymore) in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose("%s sites to be deleted" % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Site", siteName) gLogger.verbose("Deleting site %s" % siteName) if not deleteQuery["OK"]: return deleteQuery # Sites that are in CS but not (anymore) in DB toBeAdded = list(set(sitesCS).difference(set(sitesDB))) gLogger.verbose("%s site entries to be added" % len(toBeAdded)) for site in toBeAdded: query = self.rStatus.addIfNotThereStatusElement( "Site", "Status", name=site, statusType="all", status=self.defaultStatus, elementType="Site", tokenOwner=self.tokenOwner, reason="Synchronized", ) if not query["OK"]: return query return S_OK() def _syncResources(self): """ Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) """ gLogger.info("-- Synchronizing Resources --") gLogger.verbose("-> StorageElements") ses = self.__syncStorageElements() if not ses["OK"]: gLogger.error(ses["Message"]) gLogger.verbose("-> FTS") fts = self.__syncFTS() if not fts["OK"]: gLogger.error(fts["Message"]) gLogger.verbose("-> FileCatalogs") fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs["OK"]: gLogger.error(fileCatalogs["Message"]) gLogger.verbose("-> ComputingElements") computingElements = self.__syncComputingElements() if not computingElements["OK"]: gLogger.error(computingElements["Message"]) gLogger.verbose("-> removing resources that no longer exist in the CS") removingResources = self.__removeNonExistingResourcesFromRM() if not removingResources["OK"]: gLogger.error(removingResources["Message"]) return S_OK() def _syncNodes(self): """ Sync resources: compares CS with DB and does the necessary modifications. ( Queues ) """ gLogger.info("-- Synchronizing Nodes --") gLogger.verbose("-> Queues") queues = self.__syncQueues() if not queues["OK"]: gLogger.error(queues["Message"]) return S_OK() def __removeNonExistingResourcesFromRM(self): """ Remove resources from DowntimeCache table that no longer exist in the CS. """ if not getServiceURL("ResourceStatus/ResourceManagement"): gLogger.verbose( "ResourceManagement is not installed, skipping removal of non existing resources..." ) return S_OK() sesHosts = getStorageElementsHosts() if not sesHosts["OK"]: return sesHosts sesHosts = sesHosts["Value"] resources = sesHosts ftsServer = getFTS3Servers(hostOnly=True) if ftsServer["OK"]: resources.extend(ftsServer["Value"]) res = getCESiteMapping() if res["OK"]: resources.extend(list(res["Value"])) downtimes = self.rManagement.selectDowntimeCache() if not downtimes["OK"]: return downtimes # Remove hosts that no longer exist in the CS for host in downtimes["Value"]: gLogger.verbose("Checking if %s is still in the CS" % host[0]) if host[0] not in resources: gLogger.verbose("%s is no longer in CS, removing entry..." % host[0]) result = self.rManagement.deleteDowntimeCache(name=host[0]) if not result["OK"]: return result return S_OK() def __syncComputingElements(self): """ Sync ComputingElements: compares CS with DB and does the necessary modifications. """ res = getCESiteMapping() if not res["OK"]: return res cesCS = list(res["Value"]) gLogger.verbose("%s Computing elements found in CS" % len(cesCS)) cesDB = self.rStatus.selectStatusElement( "Resource", "Status", elementType="ComputingElement", meta={"columns": ["Name"]}) if not cesDB["OK"]: return cesDB cesDB = [ceDB[0] for ceDB in cesDB["Value"]] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(cesDB).difference(set(cesCS))) gLogger.verbose("%s Computing elements to be deleted" % len(toBeDeleted)) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Resource", ceName) gLogger.verbose("... %s" % ceName) if not deleteQuery["OK"]: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType("ComputingElement") result = self.rStatus.selectStatusElement( "Resource", "Status", elementType="ComputingElement", meta={"columns": ["Name", "StatusType"]}) if not result["OK"]: return result cesTuple = [(x[0], x[1]) for x in result["Value"]] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [(se, statusType) for se in cesCS for statusType in statusTypes] toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple))) gLogger.debug("%s Computing elements entries to be added" % len(toBeAdded)) for ceTuple in toBeAdded: _name = ceTuple[0] _statusType = ceTuple[1] _status = self.defaultStatus _reason = "Synchronized" _elementType = "ComputingElement" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncFileCatalogs(self): """ Sync FileCatalogs: compares CS with DB and does the necessary modifications. """ catalogsCS = CSHelpers.getFileCatalogs() if not catalogsCS["OK"]: return catalogsCS catalogsCS = catalogsCS["Value"] gLogger.verbose("%s File catalogs found in CS" % len(catalogsCS)) catalogsDB = self.rStatus.selectStatusElement( "Resource", "Status", elementType="Catalog", meta={"columns": ["Name"]}) if not catalogsDB["OK"]: return catalogsDB catalogsDB = [catalogDB[0] for catalogDB in catalogsDB["Value"]] # StorageElements that are in DB but not in CS toBeDeleted = list(set(catalogsDB).difference(set(catalogsCS))) gLogger.verbose("%s File catalogs to be deleted" % len(toBeDeleted)) # Delete storage elements for catalogName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Resource", catalogName) gLogger.verbose("... %s" % catalogName) if not deleteQuery["OK"]: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType("Catalog") result = self.rStatus.selectStatusElement( "Resource", "Status", elementType="Catalog", meta={"columns": ["Name", "StatusType"]}) if not result["OK"]: return result sesTuple = [(x[0], x[1]) for x in result["Value"]] # For each ( se, statusType ) tuple not present in the DB, add it. catalogsStatusTuples = [(se, statusType) for se in catalogsCS for statusType in statusTypes] toBeAdded = list(set(catalogsStatusTuples).difference(set(sesTuple))) gLogger.verbose("%s File catalogs entries to be added" % len(toBeAdded)) for catalogTuple in toBeAdded: _name = catalogTuple[0] _statusType = catalogTuple[1] _status = self.defaultStatus _reason = "Synchronized" _elementType = "Catalog" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncFTS(self): """ Sync FTS: compares CS with DB and does the necessary modifications. """ ftsCS = CSHelpers.getFTS() if not ftsCS["OK"]: return ftsCS ftsCS = ftsCS["Value"] gLogger.verbose("%s FTS endpoints found in CS" % len(ftsCS)) ftsDB = self.rStatus.selectStatusElement("Resource", "Status", elementType="FTS", meta={"columns": ["Name"]}) if not ftsDB["OK"]: return ftsDB ftsDB = [fts[0] for fts in ftsDB["Value"]] # StorageElements that are in DB but not in CS toBeDeleted = list(set(ftsDB).difference(set(ftsCS))) gLogger.verbose("%s FTS endpoints to be deleted" % len(toBeDeleted)) # Delete storage elements for ftsName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Resource", ftsName) gLogger.verbose("... %s" % ftsName) if not deleteQuery["OK"]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType("FTS") # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] result = self.rStatus.selectStatusElement( "Resource", "Status", elementType="FTS", meta={"columns": ["Name", "StatusType"]}) if not result["OK"]: return result sesTuple = [(x[0], x[1]) for x in result["Value"]] # For each ( se, statusType ) tuple not present in the DB, add it. ftsStatusTuples = [(se, statusType) for se in ftsCS for statusType in statusTypes] toBeAdded = list(set(ftsStatusTuples).difference(set(sesTuple))) gLogger.verbose("%s FTS endpoints entries to be added" % len(toBeAdded)) for ftsTuple in toBeAdded: _name = ftsTuple[0] _statusType = ftsTuple[1] _status = self.defaultStatus _reason = "Synchronized" _elementType = "FTS" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncStorageElements(self): """ Sync StorageElements: compares CS with DB and does the necessary modifications. """ sesCS = DMSHelpers().getStorageElements() gLogger.verbose("%s storage elements found in CS" % len(sesCS)) sesDB = self.rStatus.selectStatusElement("Resource", "Status", elementType="StorageElement", meta={"columns": ["Name"]}) if not sesDB["OK"]: return sesDB sesDB = [seDB[0] for seDB in sesDB["Value"]] # StorageElements that are in DB but not in CS toBeDeleted = list(set(sesDB).difference(set(sesCS))) gLogger.verbose("%s storage elements to be deleted" % len(toBeDeleted)) # Delete storage elements for sesName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Resource", sesName) gLogger.verbose("... %s" % sesName) if not deleteQuery["OK"]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType("StorageElement") # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] result = self.rStatus.selectStatusElement( "Resource", "Status", elementType="StorageElement", meta={"columns": ["Name", "StatusType"]}) if not result["OK"]: return result sesTuple = [(x[0], x[1]) for x in result["Value"]] # For each ( se, statusType ) tuple not present in the DB, add it. sesStatusTuples = [(se, statusType) for se in sesCS for statusType in statusTypes] toBeAdded = list(set(sesStatusTuples).difference(set(sesTuple))) gLogger.verbose("%s storage element entries to be added" % len(toBeAdded)) for seTuple in toBeAdded: _name = seTuple[0] _statusType = seTuple[1] _status = self.defaultStatus _reason = "Synchronized" _elementType = "StorageElement" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncQueues(self): """ Sync Queues: compares CS with DB and does the necessary modifications. """ queuesCS = CSHelpers.getQueuesRSS() if not queuesCS["OK"]: return queuesCS queuesCS = queuesCS["Value"] gLogger.verbose("%s Queues found in CS" % len(queuesCS)) queuesDB = self.rStatus.selectStatusElement("Node", "Status", elementType="Queue", meta={"columns": ["Name"]}) if not queuesDB["OK"]: return queuesDB queuesDB = [queueDB[0] for queueDB in queuesDB["Value"]] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(queuesDB).difference(set(queuesCS))) gLogger.verbose("%s Queues to be deleted" % len(toBeDeleted)) # Delete storage elements for queueName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Node", queueName) gLogger.verbose("... %s" % queueName) if not deleteQuery["OK"]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType("Queue") # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Node' ] result = self.rStatus.selectStatusElement( "Node", "Status", elementType="Queue", meta={"columns": ["Name", "StatusType"]}) if not result["OK"]: return result queueTuple = [(x[0], x[1]) for x in result["Value"]] # For each ( se, statusType ) tuple not present in the DB, add it. queueStatusTuples = [(se, statusType) for se in queuesCS for statusType in statusTypes] toBeAdded = list(set(queueStatusTuples).difference(set(queueTuple))) gLogger.verbose("%s Queue entries to be added" % len(toBeAdded)) for queueTuple in toBeAdded: _name = queueTuple[0] _statusType = queueTuple[1] _status = self.defaultStatus _reason = "Synchronized" _elementType = "Queue" query = self.rStatus.addIfNotThereStatusElement( "Node", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason, ) if not query["OK"]: return query return S_OK()
class Synchronizer(object): """ Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. """ def __init__(self, rStatus=None, rManagement=None): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient.ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient.ResourceManagementClient() self.rssConfig = RssConfiguration() def sync(self, _eventName, _params): """ Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK """ syncSites = self._syncSites() if not syncSites["OK"]: gLogger.error(syncSites["Message"]) syncResources = self._syncResources() if not syncResources["OK"]: gLogger.error(syncResources["Message"]) syncNodes = self._syncNodes() if not syncNodes["OK"]: gLogger.error(syncNodes["Message"]) # FIXME: also sync users return S_OK() ## Protected methods ######################################################### def _syncSites(self): """ Sync sites: compares CS with DB and does the necessary modifications. """ gLogger.info("-- Synchronizing sites --") domainSitesCS = CSHelpers.getDomainSites() if not domainSitesCS["OK"]: return domainSitesCS domainSitesCS = domainSitesCS["Value"] for domainName, sitesCS in domainSitesCS.items(): gLogger.verbose("%s sites found in CS for %s domain" % (len(sitesCS), domainName)) sitesDB = self.rStatus.selectStatusElement( "Site", "Status", elementType=domainName, meta={"columns": ["name"]} ) if not sitesDB["OK"]: return sitesDB sitesDB = [siteDB[0] for siteDB in sitesDB["Value"]] # Sites that are in DB but not in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose("%s sites to be deleted" % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement("Site", siteName) gLogger.verbose("... %s" % siteName) if not deleteQuery["OK"]: return deleteQuery sitesTuple = self.rStatus.selectStatusElement( "Site", "Status", elementType=domainName, meta={"columns": ["name", "statusType"]} ) if not sitesTuple["OK"]: return sitesTuple sitesTuple = sitesTuple["Value"] statusTypes = self.rssConfig.getConfigStatusType(domainName) # For each ( site, statusType ) tuple not present in the DB, add it. siteStatusTuples = [(site, statusType) for site in sitesCS for statusType in statusTypes] toBeAdded = list(set(siteStatusTuples).difference(set(sitesTuple))) gLogger.verbose("%s site entries to be added" % len(toBeAdded)) for siteTuple in toBeAdded: query = self.rStatus.addIfNotThereStatusElement( "Site", "Status", name=siteTuple[0], statusType=siteTuple[1], status="Unknown", elementType=domainName, reason="Synchronized", ) if not query["OK"]: return query return S_OK() def _syncResources(self): """ Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) """ gLogger.info("-- Synchronizing Resources --") gLogger.verbose("-> StorageElements") ses = self.__syncStorageElements() if not ses["OK"]: gLogger.error(ses["Message"]) gLogger.verbose("-> FTS") fts = self.__syncFTS() if not fts["OK"]: gLogger.error(fts["Message"]) gLogger.verbose("-> FileCatalogs") fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs["OK"]: gLogger.error(fileCatalogs["Message"]) gLogger.verbose("-> ComputingElements") computingElements = self.__syncComputingElements() if not computingElements["OK"]: gLogger.error(computingElements["Message"]) # FIXME: VOMS return S_OK() def _syncNodes(self): """ Sync resources: compares CS with DB and does the necessary modifications. ( Queues ) """ gLogger.info("-- Synchronizing Nodes --") gLogger.verbose("-> Queues") queues = self.__syncQueues() if not queues["OK"]: gLogger.error(queues["Message"]) return S_OK() ## Private methods ########################################################### def __syncComputingElements(self): """ Sync CEs: compares CS with DB and does the necessary modifications. """ cesCS = CSHelpers.getComputingElements() if not cesCS["OK"]: return cesCS cesCS = cesCS["Value"] gLogger.verbose("%s Computing elements found in CS" % len(cesCS)) cesDB = self.rStatus.selectStatusElement("Resource", "Status", elementType="CE", meta={"columns": ["name"]}) if not cesDB["OK"]: return cesDB cesDB = [ceDB[0] for ceDB in cesDB["Value"]] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(cesDB).difference(set(cesCS))) gLogger.verbose("%s Computing elements to be deleted" % len(toBeDeleted)) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement("Resource", ceName) gLogger.verbose("... %s" % ceName) if not deleteQuery["OK"]: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType("CE") cesTuple = self.rStatus.selectStatusElement( "Resource", "Status", elementType="CE", meta={"columns": ["name", "statusType"]} ) if not cesTuple["OK"]: return cesTuple cesTuple = cesTuple["Value"] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [(se, statusType) for se in cesCS for statusType in statusTypes] toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple))) gLogger.debug("%s Computing elements entries to be added" % len(toBeAdded)) for ceTuple in toBeAdded: _name = ceTuple[0] _statusType = ceTuple[1] _status = "Unknown" _reason = "Synchronized" _elementType = "CE" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncFileCatalogs(self): """ Sync FileCatalogs: compares CS with DB and does the necessary modifications. """ catalogsCS = CSHelpers.getFileCatalogs() if not catalogsCS["OK"]: return catalogsCS catalogsCS = catalogsCS["Value"] gLogger.verbose("%s File catalogs found in CS" % len(catalogsCS)) catalogsDB = self.rStatus.selectStatusElement( "Resource", "Status", elementType="Catalog", meta={"columns": ["name"]} ) if not catalogsDB["OK"]: return catalogsDB catalogsDB = [catalogDB[0] for catalogDB in catalogsDB["Value"]] # StorageElements that are in DB but not in CS toBeDeleted = list(set(catalogsDB).difference(set(catalogsCS))) gLogger.verbose("%s File catalogs to be deleted" % len(toBeDeleted)) # Delete storage elements for catalogName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement("Resource", catalogName) gLogger.verbose("... %s" % catalogName) if not deleteQuery["OK"]: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType("Catalog") sesTuple = self.rStatus.selectStatusElement( "Resource", "Status", elementType="Catalog", meta={"columns": ["name", "statusType"]} ) if not sesTuple["OK"]: return sesTuple sesTuple = sesTuple["Value"] # For each ( se, statusType ) tuple not present in the DB, add it. catalogsStatusTuples = [(se, statusType) for se in catalogsCS for statusType in statusTypes] toBeAdded = list(set(catalogsStatusTuples).difference(set(sesTuple))) gLogger.verbose("%s File catalogs entries to be added" % len(toBeAdded)) for catalogTuple in toBeAdded: _name = catalogTuple[0] _statusType = catalogTuple[1] _status = "Unknown" _reason = "Synchronized" _elementType = "Catalog" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncFTS(self): """ Sync FTS: compares CS with DB and does the necessary modifications. """ ftsCS = CSHelpers.getFTS() if not ftsCS["OK"]: return ftsCS ftsCS = ftsCS["Value"] gLogger.verbose("%s FTS endpoints found in CS" % len(ftsCS)) ftsDB = self.rStatus.selectStatusElement("Resource", "Status", elementType="FTS", meta={"columns": ["name"]}) if not ftsDB["OK"]: return ftsDB ftsDB = [fts[0] for fts in ftsDB["Value"]] # StorageElements that are in DB but not in CS toBeDeleted = list(set(ftsDB).difference(set(ftsCS))) gLogger.verbose("%s FTS endpoints to be deleted" % len(toBeDeleted)) # Delete storage elements for ftsName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement("Resource", ftsName) gLogger.verbose("... %s" % ftsName) if not deleteQuery["OK"]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType("FTS") # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] sesTuple = self.rStatus.selectStatusElement( "Resource", "Status", elementType="FTS", meta={"columns": ["name", "statusType"]} ) if not sesTuple["OK"]: return sesTuple sesTuple = sesTuple["Value"] # For each ( se, statusType ) tuple not present in the DB, add it. ftsStatusTuples = [(se, statusType) for se in ftsCS for statusType in statusTypes] toBeAdded = list(set(ftsStatusTuples).difference(set(sesTuple))) gLogger.verbose("%s FTS endpoints entries to be added" % len(toBeAdded)) for ftsTuple in toBeAdded: _name = ftsTuple[0] _statusType = ftsTuple[1] _status = "Unknown" _reason = "Synchronized" _elementType = "FTS" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncStorageElements(self): """ Sync StorageElements: compares CS with DB and does the necessary modifications. """ sesCS = CSHelpers.getStorageElements() if not sesCS["OK"]: return sesCS sesCS = sesCS["Value"] gLogger.verbose("%s storage elements found in CS" % len(sesCS)) sesDB = self.rStatus.selectStatusElement( "Resource", "Status", elementType="StorageElement", meta={"columns": ["name"]} ) if not sesDB["OK"]: return sesDB sesDB = [seDB[0] for seDB in sesDB["Value"]] # StorageElements that are in DB but not in CS toBeDeleted = list(set(sesDB).difference(set(sesCS))) gLogger.verbose("%s storage elements to be deleted" % len(toBeDeleted)) # Delete storage elements for sesName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement("Resource", sesName) gLogger.verbose("... %s" % sesName) if not deleteQuery["OK"]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType("StorageElement") # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] sesTuple = self.rStatus.selectStatusElement( "Resource", "Status", elementType="StorageElement", meta={"columns": ["name", "statusType"]} ) if not sesTuple["OK"]: return sesTuple sesTuple = sesTuple["Value"] # For each ( se, statusType ) tuple not present in the DB, add it. sesStatusTuples = [(se, statusType) for se in sesCS for statusType in statusTypes] toBeAdded = list(set(sesStatusTuples).difference(set(sesTuple))) gLogger.verbose("%s storage element entries to be added" % len(toBeAdded)) for seTuple in toBeAdded: _name = seTuple[0] _statusType = seTuple[1] _status = "Unknown" _reason = "Synchronized" _elementType = "StorageElement" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason, ) if not query["OK"]: return query return S_OK() def __syncQueues(self): """ Sync Queues: compares CS with DB and does the necessary modifications. """ queuesCS = CSHelpers.getQueues() if not queuesCS["OK"]: return queuesCS queuesCS = queuesCS["Value"] gLogger.verbose("%s Queues found in CS" % len(queuesCS)) queuesDB = self.rStatus.selectStatusElement("Node", "Status", elementType="Queue", meta={"columns": ["name"]}) if not queuesDB["OK"]: return queuesDB queuesDB = [queueDB[0] for queueDB in queuesDB["Value"]] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(queuesDB).difference(set(queuesCS))) gLogger.verbose("%s Queues to be deleted" % len(toBeDeleted)) # Delete storage elements for queueName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement("Node", queueName) gLogger.verbose("... %s" % queueName) if not deleteQuery["OK"]: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType("Queue") # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Node' ] queueTuple = self.rStatus.selectStatusElement( "Node", "Status", elementType="Queue", meta={"columns": ["name", "statusType"]} ) if not queueTuple["OK"]: return queueTuple queueTuple = queueTuple["Value"] # For each ( se, statusType ) tuple not present in the DB, add it. queueStatusTuples = [(se, statusType) for se in queuesCS for statusType in statusTypes] toBeAdded = list(set(queueStatusTuples).difference(set(queueTuple))) gLogger.verbose("%s Queue entries to be added" % len(toBeAdded)) for queueTuple in toBeAdded: _name = queueTuple[0] _statusType = queueTuple[1] _status = "Unknown" _reason = "Synchronized" _elementType = "Queue" query = self.rStatus.addIfNotThereStatusElement( "Node", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason, ) if not query["OK"]: return query return S_OK() def _syncUsers(self): """ Sync Users: compares CS with DB and does the necessary modifications. """ gLogger.verbose("-- Synchronizing users --") usersCS = CSHelpers.getRegistryUsers() if not usersCS["OK"]: return usersCS usersCS = usersCS["Value"] gLogger.verbose("%s users found in CS" % len(usersCS)) usersDB = self.rManagement.selectUserRegistryCache(meta={"columns": ["login"]}) if not usersDB["OK"]: return usersDB usersDB = [userDB[0] for userDB in usersDB["Value"]] # Users that are in DB but not in CS toBeDeleted = list(set(usersDB).difference(set(usersCS.keys()))) gLogger.verbose("%s users to be deleted" % len(toBeDeleted)) # Delete users # FIXME: probably it is not needed since there is a DatabaseCleanerAgent for userLogin in toBeDeleted: deleteQuery = self.rManagement.deleteUserRegistryCache(login=userLogin) gLogger.verbose("... %s" % userLogin) if not deleteQuery["OK"]: return deleteQuery # AddOrModify Users for userLogin, userDict in usersCS.items(): _name = userDict["DN"].split("=")[-1] _email = userDict["Email"] query = self.rManagement.addOrModifyUserRegistryCache(userLogin, _name, _email) gLogger.verbose("-> %s" % userLogin) if not query["OK"]: return query return S_OK()
class Synchronizer(object): ''' Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. ''' def __init__(self, rStatus=None, rManagement=None): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient.ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.rssConfig = RssConfiguration() def sync(self, _eventName, _params): ''' Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK ''' syncSites = self._syncSites() if not syncSites['OK']: gLogger.error(syncSites['Message']) syncResources = self._syncResources() if not syncResources['OK']: gLogger.error(syncResources['Message']) syncNodes = self._syncNodes() if not syncNodes['OK']: gLogger.error(syncNodes['Message']) #FIXME: also sync users return S_OK() ## Protected methods ######################################################### def _syncSites(self): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info('-- Synchronizing sites --') domainSitesCS = CSHelpers.getDomainSites() if not domainSitesCS['OK']: return domainSitesCS domainSitesCS = domainSitesCS['Value'] for domainName, sitesCS in domainSitesCS.items(): gLogger.verbose('%s sites found in CS for %s domain' % (len(sitesCS), domainName)) sitesDB = self.rStatus.selectStatusElement( 'Site', 'Status', elementType=domainName, meta={'columns': ['name']}) if not sitesDB['OK']: return sitesDB sitesDB = [siteDB[0] for siteDB in sitesDB['Value']] # Sites that are in DB but not in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose('%s sites to be deleted' % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName) gLogger.verbose('... %s' % siteName) if not deleteQuery['OK']: return deleteQuery sitesTuple = self.rStatus.selectStatusElement( 'Site', 'Status', elementType=domainName, meta={'columns': ['name', 'statusType']}) if not sitesTuple['OK']: return sitesTuple sitesTuple = sitesTuple['Value'] statusTypes = self.rssConfig.getConfigStatusType(domainName) # For each ( site, statusType ) tuple not present in the DB, add it. siteStatusTuples = [(site, statusType) for site in sitesCS for statusType in statusTypes] toBeAdded = list(set(siteStatusTuples).difference(set(sitesTuple))) gLogger.verbose('%s site entries to be added' % len(toBeAdded)) for siteTuple in toBeAdded: query = self.rStatus.addIfNotThereStatusElement( 'Site', 'Status', name=siteTuple[0], statusType=siteTuple[1], status='Unknown', elementType=domainName, reason='Synchronized') if not query['OK']: return query return S_OK() def _syncResources(self): ''' Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) ''' gLogger.info('-- Synchronizing Resources --') gLogger.verbose('-> StorageElements') ses = self.__syncStorageElements() if not ses['OK']: gLogger.error(ses['Message']) gLogger.verbose('-> FTS') fts = self.__syncFTS() if not fts['OK']: gLogger.error(fts['Message']) gLogger.verbose('-> FileCatalogs') fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs['OK']: gLogger.error(fileCatalogs['Message']) gLogger.verbose('-> ComputingElements') computingElements = self.__syncComputingElements() if not computingElements['OK']: gLogger.error(computingElements['Message']) #FIXME: VOMS return S_OK() def _syncNodes(self): ''' Sync resources: compares CS with DB and does the necessary modifications. ( Queues ) ''' gLogger.info('-- Synchronizing Nodes --') gLogger.verbose('-> Queues') queues = self.__syncQueues() if not queues['OK']: gLogger.error(queues['Message']) return S_OK() ## Private methods ########################################################### def __syncComputingElements(self): ''' Sync CEs: compares CS with DB and does the necessary modifications. ''' cesCS = CSHelpers.getComputingElements() if not cesCS['OK']: return cesCS cesCS = cesCS['Value'] gLogger.verbose('%s Computing elements found in CS' % len(cesCS)) cesDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='CE', meta={'columns': ['name']}) if not cesDB['OK']: return cesDB cesDB = [ceDB[0] for ceDB in cesDB['Value']] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(cesDB).difference(set(cesCS))) gLogger.verbose('%s Computing elements to be deleted' % len(toBeDeleted)) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ceName) gLogger.verbose('... %s' % ceName) if not deleteQuery['OK']: return deleteQuery #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType('CE') cesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='CE', meta={'columns': ['name', 'statusType']}) if not cesTuple['OK']: return cesTuple cesTuple = cesTuple['Value'] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [(se, statusType) for se in cesCS for statusType in statusTypes] toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple))) gLogger.debug('%s Computing elements entries to be added' % len(toBeAdded)) for ceTuple in toBeAdded: _name = ceTuple[0] _statusType = ceTuple[1] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'CE' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason) if not query['OK']: return query return S_OK() def __syncFileCatalogs(self): ''' Sync FileCatalogs: compares CS with DB and does the necessary modifications. ''' catalogsCS = CSHelpers.getFileCatalogs() if not catalogsCS['OK']: return catalogsCS catalogsCS = catalogsCS['Value'] gLogger.verbose('%s File catalogs found in CS' % len(catalogsCS)) catalogsDB = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='Catalog', meta={'columns': ['name']}) if not catalogsDB['OK']: return catalogsDB catalogsDB = [catalogDB[0] for catalogDB in catalogsDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(catalogsDB).difference(set(catalogsCS))) gLogger.verbose('%s File catalogs to be deleted' % len(toBeDeleted)) # Delete storage elements for catalogName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', catalogName) gLogger.verbose('... %s' % catalogName) if not deleteQuery['OK']: return deleteQuery #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType('Catalog') sesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='Catalog', meta={'columns': ['name', 'statusType']}) if not sesTuple['OK']: return sesTuple sesTuple = sesTuple['Value'] # For each ( se, statusType ) tuple not present in the DB, add it. catalogsStatusTuples = [(se, statusType) for se in catalogsCS for statusType in statusTypes] toBeAdded = list(set(catalogsStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s File catalogs entries to be added' % len(toBeAdded)) for catalogTuple in toBeAdded: _name = catalogTuple[0] _statusType = catalogTuple[1] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'Catalog' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason) if not query['OK']: return query return S_OK() def __syncFTS(self): ''' Sync FTS: compares CS with DB and does the necessary modifications. ''' ftsCS = CSHelpers.getFTS() if not ftsCS['OK']: return ftsCS ftsCS = ftsCS['Value'] gLogger.verbose('%s FTS endpoints found in CS' % len(ftsCS)) ftsDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='FTS', meta={'columns': ['name']}) if not ftsDB['OK']: return ftsDB ftsDB = [fts[0] for fts in ftsDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(ftsDB).difference(set(ftsCS))) gLogger.verbose('%s FTS endpoints to be deleted' % len(toBeDeleted)) # Delete storage elements for ftsName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', ftsName) gLogger.verbose('... %s' % ftsName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('FTS') #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] sesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='FTS', meta={'columns': ['name', 'statusType']}) if not sesTuple['OK']: return sesTuple sesTuple = sesTuple['Value'] # For each ( se, statusType ) tuple not present in the DB, add it. ftsStatusTuples = [(se, statusType) for se in ftsCS for statusType in statusTypes] toBeAdded = list(set(ftsStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s FTS endpoints entries to be added' % len(toBeAdded)) for ftsTuple in toBeAdded: _name = ftsTuple[0] _statusType = ftsTuple[1] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'FTS' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason) if not query['OK']: return query return S_OK() def __syncStorageElements(self): ''' Sync StorageElements: compares CS with DB and does the necessary modifications. ''' sesCS = CSHelpers.getStorageElements() if not sesCS['OK']: return sesCS sesCS = sesCS['Value'] gLogger.verbose('%s storage elements found in CS' % len(sesCS)) sesDB = self.rStatus.selectStatusElement('Resource', 'Status', elementType='StorageElement', meta={'columns': ['name']}) if not sesDB['OK']: return sesDB sesDB = [seDB[0] for seDB in sesDB['Value']] # StorageElements that are in DB but not in CS toBeDeleted = list(set(sesDB).difference(set(sesCS))) gLogger.verbose('%s storage elements to be deleted' % len(toBeDeleted)) # Delete storage elements for sesName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Resource', sesName) gLogger.verbose('... %s' % sesName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('StorageElement') #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] sesTuple = self.rStatus.selectStatusElement( 'Resource', 'Status', elementType='StorageElement', meta={'columns': ['name', 'statusType']}) if not sesTuple['OK']: return sesTuple sesTuple = sesTuple['Value'] # For each ( se, statusType ) tuple not present in the DB, add it. sesStatusTuples = [(se, statusType) for se in sesCS for statusType in statusTypes] toBeAdded = list(set(sesStatusTuples).difference(set(sesTuple))) gLogger.verbose('%s storage element entries to be added' % len(toBeAdded)) for seTuple in toBeAdded: _name = seTuple[0] _statusType = seTuple[1] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'StorageElement' query = self.rStatus.addIfNotThereStatusElement( 'Resource', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason) if not query['OK']: return query return S_OK() def __syncQueues(self): ''' Sync Queues: compares CS with DB and does the necessary modifications. ''' queuesCS = CSHelpers.getQueues() if not queuesCS['OK']: return queuesCS queuesCS = queuesCS['Value'] gLogger.verbose('%s Queues found in CS' % len(queuesCS)) queuesDB = self.rStatus.selectStatusElement('Node', 'Status', elementType='Queue', meta={'columns': ['name']}) if not queuesDB['OK']: return queuesDB queuesDB = [queueDB[0] for queueDB in queuesDB['Value']] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(queuesDB).difference(set(queuesCS))) gLogger.verbose('%s Queues to be deleted' % len(toBeDeleted)) # Delete storage elements for queueName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Node', queueName) gLogger.verbose('... %s' % queueName) if not deleteQuery['OK']: return deleteQuery statusTypes = self.rssConfig.getConfigStatusType('Queue') #statusTypes = RssConfiguration.getValidStatusTypes()[ 'Node' ] queueTuple = self.rStatus.selectStatusElement( 'Node', 'Status', elementType='Queue', meta={'columns': ['name', 'statusType']}) if not queueTuple['OK']: return queueTuple queueTuple = queueTuple['Value'] # For each ( se, statusType ) tuple not present in the DB, add it. queueStatusTuples = [(se, statusType) for se in queuesCS for statusType in statusTypes] toBeAdded = list(set(queueStatusTuples).difference(set(queueTuple))) gLogger.verbose('%s Queue entries to be added' % len(toBeAdded)) for queueTuple in toBeAdded: _name = queueTuple[0] _statusType = queueTuple[1] _status = 'Unknown' _reason = 'Synchronized' _elementType = 'Queue' query = self.rStatus.addIfNotThereStatusElement( 'Node', 'Status', name=_name, statusType=_statusType, status=_status, elementType=_elementType, reason=_reason) if not query['OK']: return query return S_OK() def _syncUsers(self): ''' Sync Users: compares CS with DB and does the necessary modifications. ''' gLogger.verbose('-- Synchronizing users --') usersCS = CSHelpers.getRegistryUsers() if not usersCS['OK']: return usersCS usersCS = usersCS['Value'] gLogger.verbose('%s users found in CS' % len(usersCS)) usersDB = self.rManagement.selectUserRegistryCache( meta={'columns': ['login']}) if not usersDB['OK']: return usersDB usersDB = [userDB[0] for userDB in usersDB['Value']] # Users that are in DB but not in CS toBeDeleted = list(set(usersDB).difference(set(usersCS.keys()))) gLogger.verbose('%s users to be deleted' % len(toBeDeleted)) # Delete users # FIXME: probably it is not needed since there is a DatabaseCleanerAgent for userLogin in toBeDeleted: deleteQuery = self.rManagement.deleteUserRegistryCache( login=userLogin) gLogger.verbose('... %s' % userLogin) if not deleteQuery['OK']: return deleteQuery # AddOrModify Users for userLogin, userDict in usersCS.items(): _name = userDict['DN'].split('=')[-1] _email = userDict['Email'] query = self.rManagement.addOrModifyUserRegistryCache( userLogin, _name, _email) gLogger.verbose('-> %s' % userLogin) if not query['OK']: return query return S_OK()
class Synchronizer(object): ''' Every time there is a successful write on the CS, Synchronizer().sync() is executed. It updates the database with the values on the CS. ''' def __init__(self): """ Constructor. examples: >>> s = Synchronizer() """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.operations = Operations() self.resources = Resources() self.rStatus = ResourceStatusClient.ResourceStatusClient() self.rssConfig = RssConfiguration() self.diracAdmin = DiracAdmin() def sync(self, _eventName, _params): ''' Main synchronizer method. It synchronizes the three types of elements: Sites, Resources and Nodes. Each _syncX method returns a dictionary with the additions and deletions. examples: >>> s.sync( None, None ) S_OK() :Parameters: **_eventName** - any this parameter is ignored, but needed by caller function. **_params** - any this parameter is ignored, but needed by caller function. :return: S_OK ''' defSyncResult = {'added': [], 'deleted': []} # Sites syncSites = self._syncSites() if not syncSites['OK']: self.log.error(syncSites['Message']) syncSites = (syncSites['OK'] and syncSites['Value']) or defSyncResult # Resources syncResources = self._syncResources() if not syncResources['OK']: self.log.error(syncResources['Message']) syncResources = (syncResources['OK'] and syncResources['Value']) or defSyncResult # Nodes syncNodes = self._syncNodes() if not syncNodes['OK']: self.log.error(syncNodes['Message']) syncNodes = (syncNodes['OK'] and syncNodes['Value']) or defSyncResult # Notify via email to : self.notify(syncSites, syncResources, syncNodes) return S_OK() def notify(self, syncSites, syncResources, syncNodes): """ Method sending email notification with the result of the synchronization. Email is sent to Operations( EMail/Production ) email address. examples: >>> s.notify( {}, {}, {} ) >>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} ) >>> s.notify( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, { 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} ) :Parameters: **syncSites** - dict() ( keys: added, deleted ) dictionary with the sites added and deleted from the DB **syncResources** - dict() ( keys: added, deleted ) dictionary with the resources added and deleted from the DB **syncNodes** - dict() ( keys: added, deleted ) dictionary with the nodes added and deleted from the DB :return: S_OK """ # Human readable summary msgBody = self.getBody(syncSites, syncResources, syncNodes) self.log.info(msgBody) # Email addresses toAddress = self.operations.getValue('EMail/Production', '') fromAddress = self.rssConfig.getConfigFromAddress('') if toAddress and fromAddress and msgBody: # Subject of the email setup = gConfig.getValue('DIRAC/Setup') subject = '[RSS](%s) CS Synchronization' % setup self.diracAdmin.sendMail(toAddress, subject, msgBody, fromAddress=fromAddress) def getBody(self, syncSites, syncResources, syncNodes): """ Method that given the outputs of the three synchronization methods builds a human readable string. examples: >>> s.getBody( {}, {}, {} ) '' >>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, {}, {} ) ''' SITES: Site: deleted:1 RubbishSite ''' >>> s.getBody( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] }, { 'Computing : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }}, {} ) ''' SITES: Site: deleted:1 RubbishSite RESOURCES: Computing: added:2 newCE01 newCE02 ''' :Parameters: **syncSites** - dict() ( keys: added, deleted ) dictionary with the sites added and deleted from the DB **syncResources** - dict() ( keys: added, deleted ) dictionary with the resources added and deleted from the DB **syncNodes** - dict() ( keys: added, deleted ) dictionary with the nodes added and deleted from the DB :return: str """ syncMsg = '' for element, syncResult in [('SITES', syncSites), ('RESOURCES', syncResources), ('NODES', syncNodes)]: elementsMsg = '' for elementType, elements in syncResult.items(): elementMsg = '' if elements['added']: elementMsg += '\n %s added: %d \n' % ( elementType, len(elements['added'])) elementMsg += ' ' + '\n '.join(elements['added']) if elements['deleted']: elementMsg += '\n %s deleted: %d \n' % ( elementType, len(elements['deleted'])) elementMsg += ' ' + '\n '.join(elements['deleted']) if elementMsg: elementsMsg += '\n\n%s:\n' % elementType elementsMsg += elementMsg if elementsMsg: syncMsg += '\n\n%s:' % element + elementsMsg return syncMsg #............................................................................. # Sync methods: Site, Resource & Node def _syncSites(self): """ Method that synchronizes sites ( using their canonical name: CERN.ch ) with elementType = 'Site'. It gets from the CS the eligible site names and then synchronizes them with the DB. If not on the DB, they are added. If in the DB but not on the CS, they are deleted. examples: >> s._syncSites() S_OK( { 'Site' : { 'added' : [], 'deleted' : [ 'RubbishSite' ] } } ) :return: S_OK( { 'Site' : { 'added' : [], 'deleted' : [] }} ) | S_ERROR """ # Get site names from the CS foundSites = self.resources.getEligibleSites() if not foundSites['OK']: return foundSites sites = {} # Synchronize with the DB resSync = self.__dbSync('Site', 'Site', foundSites['Value']) if not resSync['OK']: self.log.error('Error synchronizing Sites') self.log.error(resSync['Message']) else: sites = resSync['Value'] return S_OK({'Site': sites}) def _syncResources(self): """ Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary keys. It makes one sync round per key ( elementType ). Gets from the CS the eligible Resource/<elementType> names and then synchronizes them with the DB. If not on the DB, they are added. If in the DB but not on the CS, they are deleted. examples: >>> s._syncResources() S_OK( { 'Computing' : { 'added' : [ 'newCE01', 'newCE02' ], 'deleted' : [] }, 'Storage' : { 'added' : [], 'deleted' : [] }, ... } ) :return: S_OK( { 'RESOURCE_NODE_MAPPINGKey1' : { 'added' : [], 'deleted' : [] }, ...} ) """ resources = {} # Iterate over the different elementTypes for Resource ( Computing, Storage... ) for elementType in RESOURCE_NODE_MAPPING.keys(): # Get Resource / <elementType> names from CS foundResources = self.resources.getEligibleResources(elementType) if not foundResources['OK']: self.log.error(foundResources['Message']) continue # Translate CS result into a list foundResources = foundResources['Value'] # Synchronize with the DB resSync = self.__dbSync('Resource', elementType, foundResources) if not resSync['OK']: self.log.error('Error synchronizing %s %s' % ('Resource', elementType)) self.log.error(resSync['Message']) else: resources[elementType] = resSync['Value'] return S_OK(resources) def _syncNodes(self): """ Method that synchronizes resources as defined on RESOURCE_NODE_MAPPING dictionary values. It makes one sync round per key ( elementType ). Gets from the CS the eligible Node/<elementType> names and then synchronizes them with the DB. If not on the DB, they are added. If in the DB but not on the CS, they are deleted. examples: >>> s._syncNodes() S_OK( { 'Queue' : { 'added' : [], 'deleted' : [] }, ... } ) :return: S_OK( { 'RESOURCE_NODE_MAPPINGValue1' : { 'added' : [], 'deleted' : [] }, ...} ) """ nodes = {} # Iterate over the different elementTypes for Node ( Queue, AccessProtocol... ) for elementType in RESOURCE_NODE_MAPPING.values(): # Get Node / <elementType> names from CS foundNodes = self.resources.getEligibleNodes(elementType) if not foundNodes['OK']: self.log.error(foundNodes['Value']) continue # Translate CS result into a list : maps NodeName to SiteName<>NodeName to # avoid duplicates # Looong list comprehension, sorry ! foundNodes = [ '%s<>%s' % (key, item) for key, subDict in foundNodes['Value'].items() for subList in subDict.values() for item in subList ] # Synchronize with the DB resSync = self.__dbSync('Node', elementType, foundNodes) if not resSync['OK']: self.log.error('Error synchronizing %s %s' % ('Node', elementType)) self.log.error(resSync['Message']) else: nodes[elementType] = resSync['Value'] return S_OK(nodes) #............................................................................. # DB sync actions def __dbSync(self, elementFamily, elementType, elementsCS): """ Method synchronizing CS and DB. Compares <elementsCS> with <elementsDB> given the elementFamily and elementType ( e.g. Resource / Computing ). If there are missing elements in the DB, are inserted. If are missing elements in the CS, are deleted from the DB. Note that the logs from the RSS DB are kept ! ( just in case ). :Parameters: **elementFamily** - str any of the valid element families : Site, Resource, Node **elementType** - str any of the valid element types for <elementFamily> **elementsCS** - list list with the elements for <elementFamily>/<elementType> found in the CS :return: S_OK( { 'added' : [], 'deleted' : [] } ) | S_ERROR """ # deleted, added default response syncRes = { 'deleted': [], 'added': [], } # Gets <elementFamily>/<elementType> elements from DB elementsDB = self.rStatus.selectStatusElement( elementFamily, 'Status', elementType=elementType, meta={'columns': ['name']}) if not elementsDB['OK']: return elementsDB elementsDB = [elementDB[0] for elementDB in elementsDB['Value']] # Elements in DB but not in CS -> to be deleted toBeDeleted = list(set(elementsDB).difference(set(elementsCS))) if toBeDeleted: resDelete = self.__dbDelete(elementFamily, elementType, toBeDeleted) if not resDelete['OK']: return resDelete else: syncRes['deleted'] = toBeDeleted # Elements in CS but not in DB -> to be added toBeAdded = list(set(elementsCS).difference(set(elementsDB))) if toBeAdded: resInsert = self.__dbInsert(elementFamily, elementType, toBeAdded) if not resInsert['OK']: return resInsert else: syncRes['added'] = toBeAdded return S_OK(syncRes) def __dbDelete(self, elementFamily, elementType, toBeDeleted): """ Method that given the elementFamily and elementType, deletes all entries in the History and Status tables for the given elements in toBeDeleted ( all their status Types ). :Parameters: **elementFamily** - str any of the valid element families : Site, Resource, Node **elementType** - str any of the valid element types for <elementFamily>, just used for logging purposes. **toBeDeleted** - list list with the elements to be deleted :return: S_OK | S_ERROR """ self.log.info('Deleting %s %s:' % (elementFamily, elementType)) self.log.info(toBeDeleted) return self.rStatus._extermineStatusElement(elementFamily, toBeDeleted) def __dbInsert(self, elementFamily, elementType, toBeAdded): """ Method that given the elementFamily and elementType, adds all elements in toBeAdded with their respective statusTypes, obtained from the CS. They are synchronized with status 'Unknown' and reason 'Synchronized'. :Parameters: **elementFamily** - str any of the valid element families : Site, Resource, Node **elementType** - str any of the valid element types for <elementFamily> **toBeDeleted** - list list with the elements to be added :return: S_OK | S_ERROR """ self.log.info('Adding %s %s:' % (elementFamily, elementType)) self.log.info(toBeAdded) statusTypes = self.rssConfig.getConfigStatusType(elementType) for element in toBeAdded: for statusType in statusTypes: resInsert = self.rStatus.addIfNotThereStatusElement( elementFamily, 'Status', name=element, statusType=statusType, status='Unknown', elementType=elementType, reason='Synchronized') if not resInsert['OK']: return resInsert return S_OK() #............................................................................... # # def _syncUsers( self ): # ''' # Sync Users: compares CS with DB and does the necessary modifications. # ''' # # gLogger.verbose( '-- Synchronizing users --') # # usersCS = CSHelpers.getRegistryUsers() # if not usersCS[ 'OK' ]: # return usersCS # usersCS = usersCS[ 'Value' ] # # gLogger.verbose( '%s users found in CS' % len( usersCS ) ) # # usersDB = self.rManagement.selectUserRegistryCache( meta = { 'columns' : [ 'login' ] } ) # if not usersDB[ 'OK' ]: # return usersDB # usersDB = [ userDB[0] for userDB in usersDB[ 'Value' ] ] # # # Users that are in DB but not in CS # toBeDeleted = list( set( usersDB ).difference( set( usersCS.keys() ) ) ) # gLogger.verbose( '%s users to be deleted' % len( toBeDeleted ) ) # # # Delete users # # FIXME: probably it is not needed since there is a DatabaseCleanerAgent # for userLogin in toBeDeleted: # # deleteQuery = self.rManagement.deleteUserRegistryCache( login = userLogin ) # # gLogger.verbose( '... %s' % userLogin ) # if not deleteQuery[ 'OK' ]: # return deleteQuery # # # AddOrModify Users # for userLogin, userDict in usersCS.items(): # # _name = userDict[ 'DN' ].split( '=' )[ -1 ] # _email = userDict[ 'Email' ] # # query = self.rManagement.addOrModifyUserRegistryCache( userLogin, _name, _email ) # gLogger.verbose( '-> %s' % userLogin ) # if not query[ 'OK' ]: # return query # # return S_OK() ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF