def test_fixed_window_with_elastic_expiry_memcache_concurrency(self):
storage = MemcachedStorage('localhost', 11211)
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = PER_SECOND(100, 2)
def _c():
for i in range(0,50):
limiter.hit(limit)
t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c)
t1.start(), t2.start()
[t1.join(), t2.join()]
self.assertEqual(storage.get(limit.key_for()), 100)
def test_moving_window_redis(self):
storage = RedisStorage("redis://localhost:6379")
limiter = MovingWindowRateLimiter(storage)
limit = PER_SECOND(10)
start = time.time()
for i in range(0,10):
self.assertTrue(limiter.hit(limit))
time.sleep(0.095)
self.assertFalse(limiter.hit(limit))
time.sleep(0.2)
self.assertTrue(limiter.hit(limit))
self.assertTrue(limiter.hit(limit))
self.assertFalse(limiter.hit(limit))
self.assertTrue(storage.storage.llen(limit.key_for()) == 10)
def test_fixed_window_with_elastic_expiry_memcache_concurrency(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
start = int(time.time())
limit = PER_SECOND(100, 2)
def _c():
for i in range(0,50):
limiter.hit(limit)
t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c)
t1.start(), t2.start()
[t1.join(), t2.join()]
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
self.assertTrue(start + 2 <= limiter.get_window_stats(limit)[0] <= start + 3)
self.assertEqual(storage.get(limit.key_for()), 100)
def test_fixed_window_with_elastic_expiry_redis(self):
storage = RedisStorage('redis://localhost:6379')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = PER_SECOND(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_memcache(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = PER_SECOND(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
def test_in_memory_expiry(self):
with hiro.Timeline().freeze() as timeline:
storage = MemoryStorage()
limiter = FixedWindowRateLimiter(storage)
per_min = PER_MINUTE(10)
for i in range(0, 10):
self.assertTrue(limiter.hit(per_min))
timeline.forward(60)
# touch another key and yield
limiter.hit(PER_SECOND(1))
time.sleep(0.1)
self.assertTrue(per_min.key_for() not in storage.storage)
def test_moving_window_redis(self):
storage = RedisStorage("redis://localhost:6379")
limiter = MovingWindowRateLimiter(storage)
limit = PER_SECOND(10, 2)
for i in range(0,10):
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 10 - (i + 1))
time.sleep(2*0.095)
self.assertFalse(limiter.hit(limit))
time.sleep(0.4)
self.assertTrue(limiter.hit(limit))
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
def test_redis(self):
storage = RedisStorage("redis://localhost:6379")
limiter = FixedWindowRateLimiter(storage)
per_min = PER_SECOND(10)
start = time.time()
count = 0
while time.time() - start < 0.5 and count < 10:
self.assertTrue(limiter.hit(per_min))
count += 1
self.assertFalse(limiter.hit(per_min))
while time.time() - start <= 1:
time.sleep(0.1)
self.assertTrue(limiter.hit(per_min))
def test_in_memory_expiry_moving_window(self):
with hiro.Timeline().freeze() as timeline:
storage = MemoryStorage()
limiter = MovingWindowRateLimiter(storage)
per_min = PER_MINUTE(10)
per_sec = PER_SECOND(1)
for i in range(0, 2):
for i in range(0, 10):
self.assertTrue(limiter.hit(per_min))
timeline.forward(60)
self.assertTrue(limiter.hit(per_sec))
time.sleep(1)
self.assertEqual([], storage.events[per_min.key_for()])
def test_memcached(self):
storage = MemcachedStorage("memcached://localhost:11211")
limiter = FixedWindowRateLimiter(storage)
per_min = PER_SECOND(10)
start = time.time()
count = 0
while time.time() - start < 0.5 and count < 10:
self.assertTrue(limiter.hit(per_min))
count += 1
self.assertFalse(limiter.hit(per_min))
while time.time() - start <= 1:
time.sleep(0.1)
self.assertTrue(limiter.hit(per_min))
def test_fixed_window(self):
storage = MemoryStorage()
limiter = FixedWindowRateLimiter(storage)
with hiro.Timeline().freeze() as timeline:
start = int(time.time())
limit = PER_SECOND(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
timeline.forward(1)
self.assertEqual(limiter.get_window_stats(limit)[1], 10)
self.assertTrue(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_in_memory(self):
storage = MemoryStorage()
limiter = FixedWindowElasticExpiryRateLimiter(storage)
with hiro.Timeline().freeze() as timeline:
start = int(time.time())
limit = PER_SECOND(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
# three extensions to the expiry
self.assertEqual(limiter.get_window_stats(limit)[0], start + 3)
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
timeline.forward(3)
start = int(time.time())
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 9)
self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
def test_large_dataset_redis_moving_window_expiry(self):
storage = RedisStorage("redis://localhost:6379")
limiter = MovingWindowRateLimiter(storage)
limit = PER_SECOND(1000)
keys_start = storage.storage.keys('%s/*' % limit.namespace)
# 100 routes
fake_routes = [uuid4().hex for _ in range(0, 100)]
# go as fast as possible in 2 seconds.
start = time.time()
def smack(e):
while not e.is_set():
self.assertTrue(limiter.hit(limit, random.choice(fake_routes)))
events = [threading.Event() for _ in range(0, 100)]
threads = [threading.Thread(target=smack, args=(e, )) for e in events]
[k.start() for k in threads]
while time.time() - start < 2:
time.sleep(0.1)
[k.set() for k in events]
time.sleep(2)
self.assertTrue(storage.storage.keys("%s/*" % limit.namespace) == [])
