def testDeleteExpiredOnGetItem(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict._now = lambda: time() + TWO_HOURS self.assertRaises(KeyError, timedDict.__getitem__, 1) self.assertEqual([], timedDict._expirationOrder)
def testPurgeOnSetItem(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict._now = lambda : time() + TWO_HOURS timedDict[2] = SomeObject('id2') self.assertEquals([2], timedDict._expirationOrder)
def testPurgeOnSetItem(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict._now = lambda: time() + TWO_HOURS timedDict[2] = SomeObject('id2') self.assertEqual([2], timedDict._expirationOrder)
def testDeleteExpiredOnGetItem(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict._now = lambda : time() + TWO_HOURS self.assertRaises(KeyError, timedDict.__getitem__, 1) self.assertEquals([], timedDict._expirationOrder)
def testPeek(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = "Now you see me, now you don't." timedDict._now = lambda : time() + TWO_HOURS self.assertEquals("Now you see me, now you don't.", timedDict.peek(1)) self.assertRaises(KeyError, timedDict.__getitem__, 1)
def testPeek(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = "Now you see me, now you don't." timedDict._now = lambda: time() + TWO_HOURS self.assertEqual("Now you see me, now you don't.", timedDict.peek(1)) self.assertRaises(KeyError, timedDict.__getitem__, 1)
def __init__(self, timeout, name=None, secure=False, httpOnly=False): self._timeout = timeout self._store = TimedDictionary(self._timeout) self._name = '{0}{1}'.format('' if name is None else '%s-' % name, uuid4()) self._secure = secure self._httpOnly = httpOnly
def testTouch(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict[2] = SomeObject('id2') self.assertEquals([1, 2], timedDict._expirationOrder) timedDict.touch(1) self.assertEquals([2, 1], timedDict._expirationOrder)
def testTouch(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict[2] = SomeObject('id2') self.assertEqual([1, 2], timedDict._expirationOrder) timedDict.touch(1) self.assertEqual([2, 1], timedDict._expirationOrder)
def __init__(self, cacheTimeout=1 * 60 * 60, returnCachedValueInCaseOfException=False, backoffTimeout=None, **kwargs): Transparent.__init__(self, **kwargs) self._cache = TimedDictionary(timeout=cacheTimeout) self._returnCachedValueInCaseOfException = returnCachedValueInCaseOfException self._backoffTimeout = backoffTimeout self._backoffStarted = None
def testBasicContains(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[5] = 'five' timedDict['six'] = 6 self.assertTrue(5 in timedDict) self.assertTrue('six' in timedDict) self.assertFalse(42 in timedDict) self.assertFalse(None in timedDict) self.assertTrue(timedDict.has_key(5)) self.assertTrue(timedDict.has_key('six')) self.assertFalse(timedDict.has_key(42)) self.assertFalse(timedDict.has_key(None))
def testPop(self): timedDict = TimedDictionary(TWO_HOURS) setTime = time() timedDict[1] = "Now you see me, now you don't." self.assertEqual("Now you see me, now you don't.", timedDict.pop(1)) self.assertRaises(KeyError, lambda: timedDict.pop(1)) self.assertEqual("default", timedDict.pop(1, 'default')) self.assertEqual(None, timedDict.pop(1, None)) self.assertEqual("default", timedDict.pop(1, default='default')) self.assertEqual(None, timedDict.pop(1, default=None)) timedDict[1] = "Now you see me, now you don't." setTime = time() timedDict._now = lambda: setTime + TWO_HOURS + 0.000001 self.assertRaises(KeyError, lambda: timedDict.pop(1))
def testPurge(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict[2] = SomeObject('id2') timedDict._now = lambda : time() + ONE_HOUR self.assertEquals([1, 2], timedDict._expirationOrder) timedDict[3] = SomeObject('id3') timedDict.touch(2) timedDict._now = lambda : time() + TWO_HOURS timedDict.purge() self.assertEquals([3, 2], timedDict._expirationOrder) timedDict._now = lambda : time() + TWO_HOURS + TWO_HOURS timedDict.purge() self.assertEquals([], timedDict._expirationOrder)
class CookieMemoryStore(object): def __init__(self, timeout, name=None): self._timeout = timeout self._store = TimedDictionary(self._timeout) self._name = '{0}{1}'.format('' if name is None else '%s-' % name, uuid4()) def createCookie(self, anObject): cookie = str(uuid4()) self._store[cookie] = anObject return self._result(cookie, anObject) def removeCookie(self, cookie): try: del self._store[cookie] except KeyError: pass def removeCookies(self, filter): for k in self._store.keys(): try: if filter(self._store.peek(k)): del self._store[k] except (AttributeError, KeyError): pass def validateCookie(self, cookie): anObject = self._store.get(cookie) if anObject is not None: self._store.touch(cookie) return self._result(cookie, anObject) return None def cookieName(self): return self._name def _result(self, cookie, anObject): return dict( cookie=cookie, value=anObject, header='Set-Cookie: {0}={1}; path=/; expires={2}'.format( self._name, cookie, formatdate(self._now() + self._timeout))) def _now(self): return time()
def testExpiredLeastRecentlyUsedGracefullyDealtWith(self): timedDict = TimedDictionary(TWO_HOURS, lruMaxSize=2) timedDict['key1'] = 'five' timedDict['key2'] = 'six' timedDict['key3'] = 'seven' self.assertEqual(set(['key2', 'key3']), set(timedDict.keys()), set(timedDict.keys())) self.assertEqual(3, len(timedDict._times)) self.assertEqual(3, len(timedDict._expirationOrder)) timedDict.purge() self.assertEqual(2, len(timedDict._times)) self.assertEqual(2, len(timedDict._expirationOrder)) timedDict._now = lambda: time() + TWO_HOURS timedDict.purge() self.assertEqual([], list(timedDict.keys())) self.assertEqual(0, len(timedDict._times)) self.assertEqual(0, len(timedDict._expirationOrder))
def testPop(self): timedDict = TimedDictionary(TWO_HOURS) setTime = time() timedDict[1] = "Now you see me, now you don't." self.assertEqual("Now you see me, now you don't.", timedDict.pop(1)) self.assertRaises(KeyError, lambda: timedDict.pop(1)) self.assertEqual("default", timedDict.pop(1, 'default')) self.assertEqual(None, timedDict.pop(1, None)) self.assertEqual("default", timedDict.pop(1, default='default')) self.assertEqual(None, timedDict.pop(1, default=None)) timedDict[1] = "Now you see me, now you don't." setTime = time() timedDict._now = lambda : setTime + TWO_HOURS + 0.000001 self.assertRaises(KeyError, lambda: timedDict.pop(1))
def __init__(self, serviceIdentifier, flag, name=None): Observable.__init__(self, name=name) self._serviceIdentifier = serviceIdentifier self._flag = flag self._flagSet = flag.default self._seed = randomWord(20) self._cookiekey = 'flagcheck%s=' % randomWord(10) self._debugTimeout = 60 self._debugIds = TimedDictionary(self._debugTimeout)
def testBasicDeletion(self): timedDict = TimedDictionary(TWO_HOURS) timedDict['key'] = 'five' try: del timedDict['key'] except: self.fail("This shouldn't happen.") self.assertRaises(KeyError, timedDict.__delitem__, 'idontexist')
def testSize(self): timedDict = TimedDictionary(TWO_HOURS) self.assertEqual(0, timedDict.size()) timedDict['key'] = 'five' self.assertEqual(1, timedDict.size()) timedDict['key'] = 'six' self.assertEqual(1, timedDict.size()) timedDict['key2'] = 'six' self.assertEqual(2, timedDict.size()) timedDict._now = lambda: time() + TWO_HOURS self.assertEqual(0, timedDict.size())
def testExpiredLeastRecentlyUsedGracefullyDealtWith(self): timedDict = TimedDictionary(TWO_HOURS, lruMaxSize=2) timedDict['key1'] = 'five' timedDict['key2'] = 'six' timedDict['key3'] = 'seven' self.assertEquals(set(['key2', 'key3']), set(timedDict.keys()), set(timedDict.keys())) self.assertEquals(3, len(timedDict._times)) self.assertEquals(3, len(timedDict._expirationOrder)) timedDict.purge() self.assertEquals(2, len(timedDict._times)) self.assertEquals(2, len(timedDict._expirationOrder)) timedDict._now = lambda : time() + TWO_HOURS timedDict.purge() self.assertEquals([], list(timedDict.keys())) self.assertEquals(0, len(timedDict._times)) self.assertEquals(0, len(timedDict._expirationOrder))
def testBasicGetAndPut(self): timedDict = TimedDictionary(TWO_HOURS) timedDict['key'] = 'value' self.assertEquals('value', timedDict['key']) timedDict['key'] = 'otherValue' self.assertEquals('otherValue', timedDict['key']) timedDict['key'] = 5 self.assertEquals(5, timedDict['key']) timedDict['key'] = 5.0 self.assertEquals(5.0, timedDict['key']) timedDict['key'] = ['someMutable', 5] self.assertEquals(['someMutable', 5], timedDict['key']) self.assertEquals(['someMutable', 5], timedDict.get('key')) self.assertRaises(TypeError, timedDict.__setitem__, [], 'value') self.assertRaises(KeyError, timedDict.__getitem__, 'iamnothere') self.assertEquals(None, timedDict.get('iamnothere')) self.assertEquals('MARK', timedDict.get('iamnothere', 'MARK'))
def testBasicGetAndPut(self): timedDict = TimedDictionary(TWO_HOURS) timedDict['key'] = 'value' self.assertEqual('value', timedDict['key']) timedDict['key'] = 'otherValue' self.assertEqual('otherValue', timedDict['key']) timedDict['key'] = 5 self.assertEqual(5, timedDict['key']) timedDict['key'] = 5.0 self.assertEqual(5.0, timedDict['key']) timedDict['key'] = ['someMutable', 5] self.assertEqual(['someMutable', 5], timedDict['key']) self.assertEqual(['someMutable', 5], timedDict.get('key')) self.assertRaises(TypeError, timedDict.__setitem__, [], 'value') self.assertRaises(KeyError, timedDict.__getitem__, 'iamnothere') self.assertEqual(None, timedDict.get('iamnothere')) self.assertEqual('MARK', timedDict.get('iamnothere', 'MARK'))
def testBasicContains(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[5] = 'five' timedDict['six'] = 6 self.assertTrue(5 in timedDict) self.assertTrue('six' in timedDict) self.assertFalse(42 in timedDict) self.assertFalse(None in timedDict) self.assertTrue(5 in timedDict) self.assertTrue('six' in timedDict) self.assertFalse(42 in timedDict) self.assertFalse(None in timedDict)
def testLruMaxSize(self): timedDict = TimedDictionary(TWO_HOURS, lruMaxSize=2) self.assertEqual(0, timedDict.size()) timedDict['key'] = 'five' self.assertEqual(1, timedDict.size()) timedDict['key1'] = 'six' self.assertEqual(2, timedDict.size()) self.assertEqual('five', timedDict['key']) timedDict['key2'] = 'seven' self.assertEqual(2, timedDict.size()) self.assertEqual(set(['key', 'key2']), set(timedDict.keys()))
def testSize(self): timedDict = TimedDictionary(TWO_HOURS) self.assertEqual(0, timedDict.size()) timedDict['key'] = 'five' self.assertEqual(1, timedDict.size()) timedDict['key'] = 'six' self.assertEqual(1, timedDict.size()) timedDict['key2'] = 'six' self.assertEqual(2, timedDict.size()) timedDict._now = lambda : time() + TWO_HOURS self.assertEqual(0, timedDict.size())
class TimedMessageCache(Transparent): def __init__(self, cacheTimeout=1 * 60 * 60, returnCachedValueInCaseOfException=False, backoffTimeout=None, **kwargs): Transparent.__init__(self, **kwargs) self._cache = TimedDictionary(timeout=cacheTimeout) self._returnCachedValueInCaseOfException = returnCachedValueInCaseOfException self._backoffTimeout = backoffTimeout self._backoffStarted = None def clear(self): self._cache.clear() def setTimeout(self, timeout): self._cache.setTimeout(timeout) def getSize(self): return self._cache.size() def any_unknown(self, message, *args, **kwargs): found = False key = (message, repr(args), repr(kwargs)) try: value = self._cache.peek(key) except KeyError: pass else: found = True if not self._cache.hasExpired(key): return value if self._backoffStarted: if self._backoffStarted + self._backoffTimeout < now(): self._backoffStarted = None elif found: return value else: raise BackoffException() try: value = yield self.any.unknown(message, *args, **kwargs) self._cache[key] = value except (SystemExit, KeyboardInterrupt, AssertionError): raise except Exception as e: if self._backoffTimeout and isinstance(e, TimeoutException): self._backoffStarted = now() if not (self._returnCachedValueInCaseOfException and found): raise BackoffException() if not (self._returnCachedValueInCaseOfException and found): raise return value yield
class CookieMemoryStore(object): def __init__(self, timeout, name=None, secure=False, httpOnly=False): self._timeout = timeout self._store = TimedDictionary(self._timeout) self._name = '{0}{1}'.format('' if name is None else '%s-' % name, uuid4()) self._secure = secure self._httpOnly = httpOnly def createCookie(self, anObject): cookie = str(uuid4()) cookieValues = dict(value=anObject) cookieValues[SECURE] = self._secure cookieValues[HTTPONLY] = self._httpOnly self._store[cookie] = cookieValues return self._result(cookie) def removeCookie(self, cookie): try: del self._store[cookie] except KeyError: pass def removeCookies(self, filter): for k in self._store.keys(): try: if filter(self._store.peek(k)['value']): del self._store[k] except (AttributeError, KeyError): pass def validateCookie(self, cookie): cookieInfo = self._store.get(cookie) if cookieInfo is not None and cookieInfo['value'] is not None: self._store.touch(cookie) return self._result(cookie) return None def cookieName(self): return self._name def _result(self, cookie): cookieInfo = self._store.get(cookie) values = [ "{0}={1}".format(*i) for i in [(self._name, cookie), ( 'path', '/'), ('expires', formatdate(self._now() + self._timeout))] ] values.extend(k for k in [SECURE, HTTPONLY] if cookieInfo.get(k) == True) return dict(cookie=cookie, value=cookieInfo['value'], header='Set-Cookie: {0}'.format('; '.join(values))) def _now(self): return time()
def testExpirationOrderIsKeptInternally(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[3] = SomeObject(1) timedDict[1] = SomeObject(10) timedDict[2] = SomeObject(20) self.assertEqual([3, 1, 2], timedDict._expirationOrder) timedDict[1] = SomeObject(23) self.assertEqual([3, 2, 1], timedDict._expirationOrder) timedDict[0] = SomeObject(23.01) self.assertEqual([3, 2, 1, 0], timedDict._expirationOrder) del timedDict[2] self.assertEqual([3, 1, 0], timedDict._expirationOrder)
class TimedMessageCache(Transparent): def __init__(self, cacheTimeout=1*60*60, returnCachedValueInCaseOfException=False, backoffTimeout=None, **kwargs): Transparent.__init__(self, **kwargs) self._cache = TimedDictionary(timeout=cacheTimeout) self._returnCachedValueInCaseOfException = returnCachedValueInCaseOfException self._backoffTimeout = backoffTimeout self._backoffStarted = None def clear(self): self._cache.clear() def setTimeout(self, timeout): self._cache.setTimeout(timeout) def getSize(self): return self._cache.size() def any_unknown(self, message, *args, **kwargs): found = False key = (message, repr(args), repr(kwargs)) try: value = self._cache.peek(key) except KeyError: pass else: found = True if not self._cache.hasExpired(key): raise StopIteration(value) if self._backoffStarted: if self._backoffStarted + self._backoffTimeout < now(): self._backoffStarted = None elif found: raise StopIteration(value) else: raise BackoffException() try: value = yield self.any.unknown(message, *args, **kwargs) self._cache[key] = value except (SystemExit, KeyboardInterrupt, AssertionError): raise except Exception, e: if self._backoffTimeout and isinstance(e, TimeoutException): self._backoffStarted = now() if not (self._returnCachedValueInCaseOfException and found): raise BackoffException() if not (self._returnCachedValueInCaseOfException and found): raise raise StopIteration(value) yield
def testLruMaxSize(self): timedDict = TimedDictionary(TWO_HOURS, lruMaxSize=2) self.assertEqual(0, timedDict.size()) timedDict['key'] = 'five' self.assertEqual(1, timedDict.size()) timedDict['key1'] = 'six' self.assertEqual(2, timedDict.size()) self.assertEquals('five', timedDict['key']) timedDict['key2'] = 'seven' self.assertEqual(2, timedDict.size()) self.assertEquals(set(['key', 'key2']), set(timedDict.keys()))
def testExpiredOnInShouldReturnDefaultOnGetWithoutAnException(self): timedDict = TimedDictionary(TWO_HOURS) setTime = time() timedDict._now = lambda: setTime timedDict[1] = SomeObject('id1') timedDict._now = lambda: setTime + TWO_HOURS try: 1 in timedDict except KeyError: self.fail("Key shouldn't have expired yet.") except: self.fail("This should not happen.") timedDict._now = lambda: setTime + TWO_HOURS + 0.000001 self.assertEqual(None, timedDict.get(1)) self.assertEqual('a default', timedDict.get(1, 'a default')) self.assertEqual([], timedDict._expirationOrder)
def testExpiredOnInShouldReturnDefaultOnGetWithoutAnException(self): timedDict = TimedDictionary(TWO_HOURS) setTime = time() timedDict._now = lambda : setTime timedDict[1] = SomeObject('id1') timedDict._now = lambda : setTime + TWO_HOURS try: 1 in timedDict except KeyError: self.fail("Key shouldn't have expired yet.") except: self.fail("This should not happen.") timedDict._now = lambda : setTime + TWO_HOURS + 0.000001 self.assertEquals(None, timedDict.get(1)) self.assertEquals('a default', timedDict.get(1, 'a default')) self.assertEquals([], timedDict._expirationOrder)
def testPurge(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') timedDict[2] = SomeObject('id2') timedDict._now = lambda: time() + ONE_HOUR self.assertEqual([1, 2], timedDict._expirationOrder) timedDict[3] = SomeObject('id3') timedDict.touch(2) timedDict._now = lambda: time() + TWO_HOURS timedDict.purge() self.assertEqual([3, 2], timedDict._expirationOrder) timedDict._now = lambda: time() + TWO_HOURS + TWO_HOURS timedDict.purge() self.assertEqual([], timedDict._expirationOrder)
def testUpdateTimeout(self): timedDict = TimedDictionary(TWO_HOURS) timedDict['key'] = 'five' timedDict._now = lambda: time() + TWO_HOURS timedDict.setTimeout(TWO_HOURS + 1) self.assertEqual("five", timedDict["key"])
def __init__(self, timeout, name=None): self._timeout = timeout self._store = TimedDictionary(self._timeout) self._name = '{0}{1}'.format('' if name is None else '%s-' % name, uuid4())
def testGetTime(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') with stderr_replaced(): self.assertTrue(time() - timedDict.getTime(1) < 2.0)
def testClear(self): timedDict = TimedDictionary(TWO_HOURS) timedDict['key'] = 'five' timedDict.clear() self.assertRaises(KeyError, lambda: timedDict["key"])
def testUpdateTimeout(self): timedDict = TimedDictionary(TWO_HOURS) timedDict['key'] = 'five' timedDict._now = lambda : time() + TWO_HOURS timedDict.setTimeout(TWO_HOURS + 1) self.assertEquals("five", timedDict["key"])
def CookieMemoryStore(timeout, *args, **kwargs): return CookieStore( TimedDictionary(timeout), timeout, *args, **kwargs)
def __init__(self, cacheTimeout=1*60*60, returnCachedValueInCaseOfException=False, backoffTimeout=None, **kwargs): Transparent.__init__(self, **kwargs) self._cache = TimedDictionary(timeout=cacheTimeout) self._returnCachedValueInCaseOfException = returnCachedValueInCaseOfException self._backoffTimeout = backoffTimeout self._backoffStarted = None
class FlagCheck(Observable): def __init__(self, serviceIdentifier, flag, name=None): Observable.__init__(self, name=name) self._serviceIdentifier = serviceIdentifier self._flag = flag self._flagSet = flag.default self._seed = randomWord(20) self._cookiekey = 'flagcheck%s=' % randomWord(10) self._debugTimeout = 60 self._debugIds = TimedDictionary(self._debugTimeout) def updateConfig(self, this_service=None, **kwargs): state = {} if this_service is None else this_service.get('state', {}) self._flagSet = self._flag.isSet(state) return yield def handleRequest(self, arguments, Headers, **kwargs): debugId = self._getDebugId(Headers) debug = arguments.pop('debug', [None])[0] is not None or debugId if not debug: try: self._check(message='handleRequest') yield self.all.handleRequest(arguments=arguments, Headers=Headers, **kwargs) except EnvironmentError: yield 'HTTP/1.0 503 Service Temporarily Unavailable\r\n\r\n' else: debugId = self._registerDebugId(debugId=debugId, **kwargs) yield insertHeader( compose(self.all.handleRequest(arguments=arguments, Headers=Headers, **kwargs)), extraHeader='Set-Cookie: '+'; '.join([ self._cookiekey+debugId, 'Expires={0}'.format(self._zulutime().add(seconds=2*self._debugTimeout).rfc1123()), ]) ) def _zulutime(self): return ZuluTime() def _getDebugId(self, Headers): debugIds = [cookie.split(self._cookiekey)[-1].strip() for cookie in Headers.get('Cookie','').split(';') if cookie.strip().startswith(self._cookiekey)] if len(debugIds) == 1: return self._debugIds.get(debugIds[0]) return None def _registerDebugId(self, debugId, Client, **ignored): if debugId is None: clientaddress, ignoredPort = Client debugId = sha1('%s%s%s%s' % (time(), randomWord(10), clientaddress, self._seed)).hexdigest() self._debugIds[debugId] = debugId else: self._debugIds.touch(debugId) return debugId def all_unknown(self, message, **kwargs): self._check(message=message) yield self.all.unknown(message, **kwargs) def _check(self, message): if not self._flagSet: raise EnvironmentError("'%s' is not allowed at the moment (%s=False)." % (message, self._flag))