def wrapped_fcn(*args, **kwargs): # Get the lock and acquire it executionLock = LockRing().getLock("_UseUserProxy_", recursive=True) executionLock.acquire() # Check if the caller is executing with the host certificate useServerCertificate = gConfig.useServerCertificate() if useServerCertificate: gConfigurationData.setOptionInCFG( "/DIRAC/Security/UseServerCertificate", "false") try: return fcn(*args, **kwargs) except Exception as lException: # pylint: disable=broad-except value = ",".join([str(arg) for arg in lException.args]) exceptType = lException.__class__.__name__ return S_ERROR("Exception - %s: %s" % (exceptType, value)) finally: # Restore the default host certificate usage if necessary if useServerCertificate: gConfigurationData.setOptionInCFG( "/DIRAC/Security/UseServerCertificate", "true") # release the lock executionLock.release()
def _putProxy(userDN=None, userName=None, userGroup=None, vomsFlag=None, proxyFilePath=None, executionLockFlag=False): """Download proxy, place in a file and populate X509_USER_PROXY environment variable. Parameters like `userProxy` or `executeWithUserProxy`. :returns: Tuple of originalUserProxy, useServerCertificate, executionLock """ # Setup user proxy if userDN: userDNs = [userDN] else: result = getDNForUsername(userName) if not result["OK"]: return result userDNs = result["Value"] # a same user may have more than one DN vomsAttr = "" if vomsFlag: vomsAttr = getVOMSAttributeForGroup(userGroup) result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath) if not result["OK"]: return result executionLock = LockRing().getLock( "_UseUserProxy_", recursive=True) if executionLockFlag else None if executionLockFlag: executionLock.acquire() os.environ["X509_USER_PROXY"], originalUserProxy = result[ "Value"], os.environ.get("X509_USER_PROXY") # Check if the caller is executing with the host certificate useServerCertificate = gConfig.useServerCertificate() if useServerCertificate: gConfigurationData.setOptionInCFG( "/DIRAC/Security/UseServerCertificate", "false") return S_OK((originalUserProxy, useServerCertificate, executionLock))
def wrapped_fcn( *args, **kwargs ): # Get the lock and acquire it executionLock = LockRing().getLock( '_UseUserProxy_', recursive = True ) executionLock.acquire() # Check if the caller is executing with the host certificate useServerCertificate = gConfig.useServerCertificate() if useServerCertificate: gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' ) try: return fcn( *args, **kwargs ) except Exception as lException: # pylint: disable=broad-except value = ','.join( [str( arg ) for arg in lException.args] ) exceptType = lException.__class__.__name__ return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) ) finally: # Restore the default host certificate usage if necessary if useServerCertificate: gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' ) # release the lock executionLock.release()
def _putProxy(userDN=None, userName=None, userGroup=None, vomsFlag=None, proxyFilePath=None, executionLockFlag=False): """Download proxy, place in a file and populate X509_USER_PROXY environment variable. Parameters like `userProxy` or `executeWithUserProxy`. :returns: Tuple of originalUserProxy, useServerCertificate, executionLock """ # Setup user proxy if userDN: userDNs = [userDN] else: result = getDNForUsername(userName) if not result['OK']: return result userDNs = result['Value'] # a same user may have more than one DN vomsAttr = '' if vomsFlag: vomsAttr = getVOMSAttributeForGroup(userGroup) result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath) if not result['OK']: return result executionLock = LockRing().getLock('_UseUserProxy_', recursive=True) if executionLockFlag else None if executionLockFlag: executionLock.acquire() os.environ['X509_USER_PROXY'], originalUserProxy = result['Value'], os.environ.get('X509_USER_PROXY') # Check if the caller is executing with the host certificate useServerCertificate = gConfig.useServerCertificate() if useServerCertificate: gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false') return S_OK((originalUserProxy, useServerCertificate, executionLock))
class Cache(object): """ Cache basic class. WARNING: None of its methods is thread safe. Acquire / Release lock when using them ! """ def __init__(self, lifeTime, updateFunc): """ Constructor :Parameters: **lifeTime** - `int` Lifetime of the elements in the cache ( seconds ! ) **updateFunc** - `function` This function MUST return a S_OK | S_ERROR object. In the case of the first, its value must be a dictionary. """ # We set a 20% of the lifetime randomly, so that if we have thousands of jobs # starting at the same time, all the caches will not end at the same time. randomLifeTimeBias = 0.2 * random.random() self.log = gLogger.getSubLogger(self.__class__.__name__) self.__lifeTime = int(lifeTime * (1 + randomLifeTimeBias)) self.__updateFunc = updateFunc # The records returned from the cache must be valid at least 30 seconds. self.__validSeconds = 30 # Cache self.__cache = DictCache() self.__cacheLock = LockRing() self.__cacheLock.getLock(self.__class__.__name__) #............................................................................. # internal cache object getter def cacheKeys(self): """ Cache keys getter :returns: list with valid keys on the cache """ return self.__cache.getKeys(validSeconds=self.__validSeconds) #............................................................................. # acquire / release Locks def acquireLock(self): """ Acquires Cache lock """ self.__cacheLock.acquire(self.__class__.__name__) def releaseLock(self): """ Releases Cache lock """ self.__cacheLock.release(self.__class__.__name__) #............................................................................. # Cache getters def get(self, cacheKeys): """ Gets values for cacheKeys given, if all are found ( present on the cache and valid ), returns S_OK with the results. If any is not neither present not valid, returns S_ERROR. :Parameters: **cacheKeys** - `list` list of keys to be extracted from the cache :return: S_OK | S_ERROR """ result = {} for cacheKey in cacheKeys: cacheRow = self.__cache.get(cacheKey, validSeconds=self.__validSeconds) if not cacheRow: self.log.error(str(cacheKey)) return S_ERROR('Cannot get %s' % str(cacheKey)) result.update({cacheKey: cacheRow}) return S_OK(result) #............................................................................. # Cache refreshers def refreshCache(self): """ Purges the cache and gets fresh data from the update function. :return: S_OK | S_ERROR. If the first, its content is the new cache. """ self.log.verbose('refreshing...') self.__cache.purgeAll() newCache = self.__updateFunc() if not newCache['OK']: self.log.error(newCache['Message']) return newCache newCache = self.__updateCache(newCache['Value']) self.log.verbose('refreshed') return newCache #............................................................................. # Private methods def __updateCache(self, newCache): """ Given the new cache dictionary, updates the internal cache with it. It sets a duration to the entries of <self.__lifeTime> seconds. :Parameters: **newCache** - `dict` dictionary containing a new cache :return: dictionary. It is newCache argument. """ for cacheKey, cacheValue in newCache.items(): self.__cache.add(cacheKey, self.__lifeTime, value=cacheValue) # We are assuming nothing will fail while inserting in the cache. There is # no apparent reason to suspect from that piece of code. return S_OK(newCache)
class MessageForwarder(object): def __init__( self, msgBroker ): self.__inOutLock = LockRing().getLock() self.__msgBroker = msgBroker self.__byClient = {} self.__srvToCliTrid = {} def addClient( self, cliTrid, destination, clientInitParams, connectParams ): if cliTrid in self.__byClient: gLogger.fatal( "Trid is duplicated!! this shouldn't happen" ) return msgClient = MessageClient( destination, **clientInitParams ) msgClient.subscribeToDisconnect( self.__srvDisconnect ) msgClient.subscribeToAllMessages( self.msgFromSrv ) msgClient.setUniqueName( connectParams[0] ) result = msgClient.connect( **connectParams[1] ) if not result[ 'OK' ]: return result self.__inOutLock.acquire() try: self.__byClient[ cliTrid ] = { 'srvEnd' : msgClient, 'srvTrid' : msgClient.getTrid(), 'srvName' : destination } self.__srvToCliTrid[ msgClient.getTrid() ] = cliTrid finally: self.__inOutLock.release() return result def __srvDisconnect( self, srvEndCli ): try: cliTrid = self.__srvToCliTrid[ srvEndCli.getTrid() ] except IndexError: gLogger.exception( "This shouldn't happen!" ) gLogger.info( "Service %s disconnected messaging connection" % self.__byClient[ cliTrid ][ 'srvName' ] ) self.__msgBroker.removeTransport( cliTrid ) self.__removeClient( cliTrid ) def cliDisconnect( self, cliTrid ): if cliTrid not in self.__byClient: gLogger.fatal( "This shouldn't happen!" ) return gLogger.info( "Client to %s disconnected messaging connection" % self.__byClient[ cliTrid ][ 'srvName' ] ) self.__byClient[ cliTrid ][ 'srvEnd' ].disconnect() self.__removeClient( cliTrid ) def __removeClient( self, cliTrid ): self.__inOutLock.acquire() try: try: srvTrid = self.__byClient[ cliTrid ][ 'srvTrid' ] self.__byClient.pop( cliTrid ) self.__srvToCliTrid.pop( srvTrid ) except Exception as e: gLogger.exception( "This shouldn't happen!" ) finally: self.__inOutLock.release() def msgFromClient( self, cliTrid, msgObj ): gLogger.info( "Message %s to %s service" % ( msgObj.getName(), self.__byClient[ cliTrid ][ 'srvName' ] ) ) result = self.__byClient[ cliTrid ][ 'srvEnd' ].sendMessage( msgObj ) return result def msgFromSrv( self, srvEndCli, msgObj ): try: cliTrid = self.__srvToCliTrid[ srvEndCli.getTrid() ] except: gLogger.exception( "This shouldn't happen" ) return S_ERROR( "MsgFromSrv -> Mismatched srv2cli trid" ) gLogger.info( "Message %s from %s service" % ( msgObj.getName(), self.__byClient[ cliTrid ][ 'srvName' ] ) ) return self.__msgBroker.sendMessage( cliTrid, msgObj )
class Cache( object ): """ Cache basic class. WARNING: None of its methods is thread safe. Acquire / Release lock when using them ! """ def __init__( self, lifeTime, updateFunc ): """ Constructor :Parameters: **lifeTime** - `int` Lifetime of the elements in the cache ( seconds ! ) **updateFunc** - `function` This function MUST return a S_OK | S_ERROR object. In the case of the first, its value must be a dictionary. """ # We set a 20% of the lifetime randomly, so that if we have thousands of jobs # starting at the same time, all the caches will not end at the same time. randomLifeTimeBias = 0.2 * random.random() self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.__lifeTime = int( lifeTime * ( 1 + randomLifeTimeBias ) ) self.__updateFunc = updateFunc # The records returned from the cache must be valid at least 10 seconds. self.__validSeconds = 10 # Cache self.__cache = DictCache() self.__cacheLock = LockRing() self.__cacheLock.getLock( self.__class__.__name__ ) #............................................................................. # internal cache object getter def cacheKeys( self ): """ Cache keys getter :returns: list with valid keys on the cache """ return self.__cache.getKeys( validSeconds = self.__validSeconds ) #............................................................................. # acquire / release Locks def acquireLock( self ): """ Acquires Cache lock """ self.__cacheLock.acquire( self.__class__.__name__ ) def releaseLock( self ): """ Releases Cache lock """ self.__cacheLock.release( self.__class__.__name__) #............................................................................. # Cache getters def get( self, cacheKeys ): """ Gets values for cacheKeys given, if all are found ( present on the cache and valid ), returns S_OK with the results. If any is not neither present not valid, returns S_ERROR. :Parameters: **cacheKeys** - `list` list of keys to be extracted from the cache :return: S_OK | S_ERROR """ result = {} for cacheKey in cacheKeys: cacheRow = self.__cache.get( cacheKey, validSeconds = self.__validSeconds ) if not cacheRow: self.log.error( str( cacheKey ) ) return S_ERROR( 'Cannot get %s' % str( cacheKey ) ) result.update( { cacheKey : cacheRow } ) return S_OK( result ) #............................................................................. # Cache refreshers def refreshCache( self ): """ Purges the cache and gets fresh data from the update function. :return: S_OK | S_ERROR. If the first, its content is the new cache. """ self.log.verbose( 'refreshing...' ) self.__cache.purgeAll() newCache = self.__updateFunc() if not newCache[ 'OK' ]: self.log.error( newCache[ 'Message' ] ) return newCache newCache = self.__updateCache( newCache[ 'Value' ] ) self.log.verbose( 'refreshed' ) return newCache #............................................................................. # Private methods def __updateCache( self, newCache ): """ Given the new cache dictionary, updates the internal cache with it. It sets a duration to the entries of <self.__lifeTime> seconds. :Parameters: **newCache** - `dict` dictionary containing a new cache :return: dictionary. It is newCache argument. """ for cacheKey, cacheValue in newCache.items(): self.__cache.add( cacheKey, self.__lifeTime, value = cacheValue ) # We are assuming nothing will fail while inserting in the cache. There is # no apparent reason to suspect from that piece of code. return S_OK( newCache )
class MessageForwarder(object): def __init__(self, msgBroker): self.__inOutLock = LockRing().getLock() self.__msgBroker = msgBroker self.__byClient = {} self.__srvToCliTrid = {} def addClient(self, cliTrid, destination, clientInitParams, connectParams): if cliTrid in self.__byClient: gLogger.fatal("Trid is duplicated!! this shouldn't happen") return msgClient = MessageClient(destination, **clientInitParams) msgClient.subscribeToDisconnect(self.__srvDisconnect) msgClient.subscribeToAllMessages(self.msgFromSrv) msgClient.setUniqueName(connectParams[0]) result = msgClient.connect(**connectParams[1]) if not result["OK"]: return result self.__inOutLock.acquire() try: self.__byClient[cliTrid] = {"srvEnd": msgClient, "srvTrid": msgClient.getTrid(), "srvName": destination} self.__srvToCliTrid[msgClient.getTrid()] = cliTrid finally: self.__inOutLock.release() return result def __srvDisconnect(self, srvEndCli): try: cliTrid = self.__srvToCliTrid[srvEndCli.getTrid()] except IndexError: gLogger.exception("This shouldn't happen!") gLogger.info("Service %s disconnected messaging connection" % self.__byClient[cliTrid]["srvName"]) self.__msgBroker.removeTransport(cliTrid) self.__removeClient(cliTrid) def cliDisconnect(self, cliTrid): if cliTrid not in self.__byClient: gLogger.fatal("This shouldn't happen!") return gLogger.info("Client to %s disconnected messaging connection" % self.__byClient[cliTrid]["srvName"]) self.__byClient[cliTrid]["srvEnd"].disconnect() self.__removeClient(cliTrid) def __removeClient(self, cliTrid): self.__inOutLock.acquire() try: try: srvTrid = self.__byClient[cliTrid]["srvTrid"] self.__byClient.pop(cliTrid) self.__srvToCliTrid.pop(srvTrid) except Exception as e: gLogger.exception("This shouldn't happen!", e) finally: self.__inOutLock.release() def msgFromClient(self, cliTrid, msgObj): gLogger.info("Message %s to %s service" % (msgObj.getName(), self.__byClient[cliTrid]["srvName"])) result = self.__byClient[cliTrid]["srvEnd"].sendMessage(msgObj) return result def msgFromSrv(self, srvEndCli, msgObj): try: cliTrid = self.__srvToCliTrid[srvEndCli.getTrid()] except Exception: gLogger.exception("This shouldn't happen") return S_ERROR("MsgFromSrv -> Mismatched srv2cli trid") gLogger.info("Message %s from %s service" % (msgObj.getName(), self.__byClient[cliTrid]["srvName"])) return self.__msgBroker.sendMessage(cliTrid, msgObj)
class DictCache: def __init__( self, deleteFunction = False ): """ Initialize the dict cache. If a delete function is specified it will be invoked when deleting a cached object """ self.__lock = LockRing() self.__lock.getLock( self.__class__.__name__, recursive = True ) self.__cache = {} self.__deleteFunction = deleteFunction def exists( self, cKey, validSeconds = 0 ): """ Returns True/False if the key exists for the given number of seconds Arguments: - cKey : identification key of the record - validSeconds : The amount of seconds the key has to be valid for """ self.__lock.acquire( self.__class__.__name__ ) try: #Is the key in the cache? if cKey in self.__cache: expTime = self.__cache[ cKey ][ 'expirationTime' ] #If it's valid return True! if expTime > datetime.datetime.now() + datetime.timedelta( seconds = validSeconds ): return True else: #Delete expired self.delete( cKey ) return False finally: self.__lock.release( self.__class__.__name__ ) def delete( self, cKey ): """ Delete a key from the cache Arguments: - cKey : identification key of the record """ self.__lock.acquire( self.__class__.__name__ ) try: if cKey not in self.__cache: return if self.__deleteFunction: self.__deleteFunction( self.__cache[ cKey ][ 'value' ] ) del( self.__cache[ cKey ] ) finally: self.__lock.release( self.__class__.__name__ ) def add( self, cKey, validSeconds, value = None ): """ Add a record to the cache Arguments: - cKey : identification key of the record - validSeconds : valid seconds of this record - value : value of the record """ if max( 0, validSeconds ) == 0: return self.__lock.acquire( self.__class__.__name__ ) try: vD = { 'expirationTime' : datetime.datetime.now() + datetime.timedelta( seconds = validSeconds ), 'value' : value } self.__cache[ cKey ] = vD finally: self.__lock.release( self.__class__.__name__ ) def get( self, cKey, validSeconds = 0 ): """ Get a record from the cache Arguments: - cKey : identification key of the record - validSeconds : The amount of seconds the key has to be valid for """ self.__lock.acquire( self.__class__.__name__ ) try: #Is the key in the cache? if cKey in self.__cache: expTime = self.__cache[ cKey ][ 'expirationTime' ] #If it's valid return True! if expTime > datetime.datetime.now() + datetime.timedelta( seconds = validSeconds ): return self.__cache[ cKey ][ 'value' ] else: #Delete expired self.delete( cKey ) return False finally: self.__lock.release( self.__class__.__name__ ) def showContentsInString( self ): """ Return a human readable string to represent the contents """ self.__lock.acquire( self.__class__.__name__ ) try: data = [] for cKey in self.__cache: data.append( "%s:" % str( cKey ) ) data.append( "\tExp: %s" % self.__cache[ cKey ][ 'expirationTime' ] ) if self.__cache[ cKey ][ 'value' ]: data.append( "\tVal: %s" % self.__cache[ cKey ][ 'value' ] ) return "\n".join( data ) finally: self.__lock.release( self.__class__.__name__ ) def getKeys( self, validSeconds = 0 ): """ Get keys for all contents """ self.__lock.acquire( self.__class__.__name__ ) try: keys = [] limitTime = datetime.datetime.now() + datetime.timedelta( seconds = validSeconds ) for cKey in self.__cache: if self.__cache[ cKey ][ 'expirationTime' ] > limitTime: keys.append( cKey ) return keys finally: self.__lock.release( self.__class__.__name__ ) def purgeExpired( self, expiredInSeconds = 0 ): """ Purge all entries that are expired or will be expired in <expiredInSeconds> """ self.__lock.acquire( self.__class__.__name__ ) try: keys = [] limitTime = datetime.datetime.now() + datetime.timedelta( seconds = expiredInSeconds ) for cKey in self.__cache: if self.__cache[ cKey ][ 'expirationTime' ] < limitTime: keys.append( cKey ) for cKey in keys: if self.__deleteFunction: self.__deleteFunction( self.__cache[ cKey ][ 'value' ] ) del( self.__cache[ cKey ] ) finally: self.__lock.release( self.__class__.__name__ ) def purgeAll( self ): """ Purge all entries """ self.__lock.acquire( self.__class__.__name__ ) try: keys = self.__cache.keys() for cKey in keys: if self.__deleteFunction: self.__deleteFunction( self.__cache[ cKey ][ 'value' ] ) del( self.__cache[ cKey ] ) finally: self.__lock.release( self.__class__.__name__)
def wrapped_fcn(*args, **kwargs): userName = kwargs.pop('proxyUserName', '') userDN = kwargs.pop('proxyUserDN', '') userGroup = kwargs.pop('proxyUserGroup', '') vomsFlag = kwargs.pop('proxyWithVOMS', True) proxyFilePath = kwargs.pop('proxyFilePath', False) executionLockFlag = kwargs.pop('executionLock', False) if executionLockFlag: executionLock = LockRing().getLock('_UseUserProxy_', recursive=True) if (userName or userDN) and userGroup: # Setup user proxy originalUserProxy = os.environ.get('X509_USER_PROXY') if userDN: userDNs = [userDN] else: result = getDNForUsername(userName) if not result['OK']: return result userDNs = result[ 'Value'] # a same user may have more than one DN vomsAttr = '' if vomsFlag: vomsAttr = getVOMSAttributeForGroup(userGroup) result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath) if not result['OK']: return result if executionLockFlag: executionLock.acquire() proxyFile = result['Value'] os.environ['X509_USER_PROXY'] = proxyFile # Check if the caller is executing with the host certificate useServerCertificate = gConfig.useServerCertificate() if useServerCertificate: gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false') try: return fcn(*args, **kwargs) except Exception as lException: # pylint: disable=broad-except value = ','.join([str(arg) for arg in lException.args]) exceptType = lException.__class__.__name__ return S_ERROR("Exception - %s: %s" % (exceptType, value)) finally: # Restore the default host certificate usage if necessary if useServerCertificate: gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true') if originalUserProxy: os.environ['X509_USER_PROXY'] = originalUserProxy else: os.environ.pop('X509_USER_PROXY') if executionLockFlag: executionLock.release() else: # No proxy substitution requested return fcn(*args, **kwargs)
class Cache: """ Cache basic class. WARNING: None of its methods is thread safe. Acquire / Release lock when using them ! """ def __init__(self, lifeTime, updateFunc): """ Constructor :Parameters: **lifeTime** - `int` Lifetime of the elements in the cache ( seconds ! ) **updateFunc** - `function` This function MUST return a S_OK | S_ERROR object. In the case of the first, its value must be a dictionary. """ # We set a 20% of the lifetime randomly, so that if we have thousands of jobs # starting at the same time, all the caches will not end at the same time. randomLifeTimeBias = 0.2 * random.random() self.log = gLogger.getSubLogger(self.__class__.__name__) self.__lifeTime = int(lifeTime * (1 + randomLifeTimeBias)) self.__updateFunc = updateFunc # The records returned from the cache must be valid at least 30 seconds. self.__validSeconds = 30 # Cache self.__cache = DictCache() self.__cacheLock = LockRing() self.__cacheLock.getLock(self.__class__.__name__) # internal cache object getter def cacheKeys(self): """ Cache keys getter :returns: list with keys in the cache valid for at least twice the validity period of the element """ # Here we need to have more than the validity period because of the logic of the matching: # * get all the keys with validity T # * for each key K, get the element K with validity T # This logic fails for elements just at the limit of the required time return self.__cache.getKeys(validSeconds=self.__validSeconds * 2) # acquire / release Locks def acquireLock(self): """ Acquires Cache lock """ self.__cacheLock.acquire(self.__class__.__name__) def releaseLock(self): """ Releases Cache lock """ self.__cacheLock.release(self.__class__.__name__) # Cache getters def get(self, cacheKeys): """ Gets values for cacheKeys given, if all are found ( present on the cache and valid ), returns S_OK with the results. If any is not neither present not valid, returns S_ERROR. :Parameters: **cacheKeys** - `list` list of keys to be extracted from the cache :return: S_OK | S_ERROR """ result = {} for cacheKey in cacheKeys: cacheRow = self.__cache.get(cacheKey, validSeconds=self.__validSeconds) if not cacheRow: return S_ERROR("Cannot get %s" % str(cacheKey)) result.update({cacheKey: cacheRow}) return S_OK(result) def check(self, cacheKeys, vO): """ Modified get() method. Attempts to find keys with a vO value appended or 'all' value appended. The cacheKeys passed in are 'flattened' cache keys (no vO) Gets values for cacheKeys given, if all are found ( present on the cache and valid ), returns S_OK with the results. If any is not neither present not valid, returns S_ERROR. :Parameters: **cacheKeys** - `list` list of keys to be extracted from the cache :return: S_OK | S_ERROR """ result = {} for cacheKey in cacheKeys: longCacheKey = cacheKey + ("all", ) cacheRow = self.__cache.get(longCacheKey, validSeconds=self.__validSeconds) if not cacheRow: longCacheKey = cacheKey + (vO, ) cacheRow = self.__cache.get(longCacheKey, validSeconds=self.__validSeconds) if not cacheRow: return S_ERROR( 'Cannot get extended %s (neither for VO = %s nor for "all" Vos)' % (str(cacheKey), vO)) result.update({longCacheKey: cacheRow}) return S_OK(result) # Cache refreshers def refreshCache(self): """ Purges the cache and gets fresh data from the update function. :return: S_OK | S_ERROR. If the first, its content is the new cache. """ self.log.verbose("refreshing...") self.__cache.purgeAll() newCache = self.__updateFunc() if not newCache["OK"]: self.log.error(newCache["Message"]) return newCache newCache = self.__updateCache(newCache["Value"]) self.log.verbose("refreshed") return newCache # Private methods def __updateCache(self, newCache): """ Given the new cache dictionary, updates the internal cache with it. It sets a duration to the entries of <self.__lifeTime> seconds. :Parameters: **newCache** - `dict` dictionary containing a new cache :return: dictionary. It is newCache argument. """ for cacheKey, cacheValue in newCache.items(): self.__cache.add(cacheKey, self.__lifeTime, value=cacheValue) # We are assuming nothing will fail while inserting in the cache. There is # no apparent reason to suspect from that piece of code. return S_OK(newCache)
class DictCache: def __init__(self, deleteFunction=False): """ Initialize the dict cache. If a delete function is specified it will be invoked when deleting a cached object """ self.__lock = LockRing() self.__lock.getLock(self.__class__.__name__, recursive=True) self.__cache = {} self.__deleteFunction = deleteFunction def exists(self, cKey, validSeconds=0): """ Returns True/False if the key exists for the given number of seconds Arguments: - cKey : identification key of the record - validSeconds : The amount of seconds the key has to be valid for """ self.__lock.acquire(self.__class__.__name__) try: #Is the key in the cache? if cKey in self.__cache: expTime = self.__cache[cKey]['expirationTime'] #If it's valid return True! if expTime > datetime.datetime.now() + datetime.timedelta( seconds=validSeconds): return True else: #Delete expired self.delete(cKey) return False finally: self.__lock.release(self.__class__.__name__) def delete(self, cKey): """ Delete a key from the cache Arguments: - cKey : identification key of the record """ self.__lock.acquire(self.__class__.__name__) try: if cKey not in self.__cache: return if self.__deleteFunction: self.__deleteFunction(self.__cache[cKey]['value']) del (self.__cache[cKey]) finally: self.__lock.release(self.__class__.__name__) def add(self, cKey, validSeconds, value=None): """ Add a record to the cache Arguments: - cKey : identification key of the record - validSeconds : valid seconds of this record - value : value of the record """ if max(0, validSeconds) == 0: return self.__lock.acquire(self.__class__.__name__) try: vD = { 'expirationTime': datetime.datetime.now() + datetime.timedelta(seconds=validSeconds), 'value': value } self.__cache[cKey] = vD finally: self.__lock.release(self.__class__.__name__) def get(self, cKey, validSeconds=0): """ Get a record from the cache Arguments: - cKey : identification key of the record - validSeconds : The amount of seconds the key has to be valid for """ self.__lock.acquire(self.__class__.__name__) try: #Is the key in the cache? if cKey in self.__cache: expTime = self.__cache[cKey]['expirationTime'] #If it's valid return True! if expTime > datetime.datetime.now() + datetime.timedelta( seconds=validSeconds): return self.__cache[cKey]['value'] else: #Delete expired self.delete(cKey) return False finally: self.__lock.release(self.__class__.__name__) def showContentsInString(self): """ Return a human readable string to represent the contents """ self.__lock.acquire(self.__class__.__name__) try: data = [] for cKey in self.__cache: data.append("%s:" % str(cKey)) data.append("\tExp: %s" % self.__cache[cKey]['expirationTime']) if self.__cache[cKey]['value']: data.append("\tVal: %s" % self.__cache[cKey]['value']) return "\n".join(data) finally: self.__lock.release(self.__class__.__name__) def getKeys(self, validSeconds=0): """ Get keys for all contents """ self.__lock.acquire(self.__class__.__name__) try: keys = [] limitTime = datetime.datetime.now() + datetime.timedelta( seconds=validSeconds) for cKey in self.__cache: if self.__cache[cKey]['expirationTime'] > limitTime: keys.append(cKey) return keys finally: self.__lock.release(self.__class__.__name__) def purgeExpired(self, expiredInSeconds=0): """ Purge all entries that are expired or will be expired in <expiredInSeconds> """ self.__lock.acquire(self.__class__.__name__) try: keys = [] limitTime = datetime.datetime.now() + datetime.timedelta( seconds=expiredInSeconds) for cKey in self.__cache: if self.__cache[cKey]['expirationTime'] < limitTime: keys.append(cKey) for cKey in keys: if self.__deleteFunction: self.__deleteFunction(self.__cache[cKey]['value']) del (self.__cache[cKey]) finally: self.__lock.release(self.__class__.__name__) def purgeAll(self): """ Purge all entries """ self.__lock.acquire(self.__class__.__name__) try: keys = self.__cache.keys() for cKey in keys: if self.__deleteFunction: self.__deleteFunction(self.__cache[cKey]['value']) del (self.__cache[cKey]) finally: self.__lock.release(self.__class__.__name__)