def test_fixed_window_with_elastic_expiry_memcache_concurrency(self): storage = MemcachedStorage('localhost', 11211) limiter = FixedWindowElasticExpiryRateLimiter(storage) limit = PER_SECOND(100, 2) def _c(): for i in range(0,50): limiter.hit(limit) t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c) t1.start(), t2.start() [t1.join(), t2.join()] self.assertEqual(storage.get(limit.key_for()), 100)
def test_moving_window_redis(self): storage = RedisStorage("redis://localhost:6379") limiter = MovingWindowRateLimiter(storage) limit = PER_SECOND(10) start = time.time() for i in range(0,10): self.assertTrue(limiter.hit(limit)) time.sleep(0.095) self.assertFalse(limiter.hit(limit)) time.sleep(0.2) self.assertTrue(limiter.hit(limit)) self.assertTrue(limiter.hit(limit)) self.assertFalse(limiter.hit(limit)) self.assertTrue(storage.storage.llen(limit.key_for()) == 10)
def test_fixed_window_with_elastic_expiry_memcache_concurrency(self): storage = MemcachedStorage('memcached://localhost:11211') limiter = FixedWindowElasticExpiryRateLimiter(storage) start = int(time.time()) limit = PER_SECOND(100, 2) def _c(): for i in range(0,50): limiter.hit(limit) t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c) t1.start(), t2.start() [t1.join(), t2.join()] self.assertEqual(limiter.get_window_stats(limit)[1], 0) self.assertTrue(start + 2 <= limiter.get_window_stats(limit)[0] <= start + 3) self.assertEqual(storage.get(limit.key_for()), 100)
def test_fixed_window_with_elastic_expiry_redis(self): storage = RedisStorage('redis://localhost:6379') limiter = FixedWindowElasticExpiryRateLimiter(storage) limit = PER_SECOND(10, 2) self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)])) time.sleep(1) self.assertFalse(limiter.hit(limit)) time.sleep(1) self.assertFalse(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_memcache(self): storage = MemcachedStorage('memcached://localhost:11211') limiter = FixedWindowElasticExpiryRateLimiter(storage) limit = PER_SECOND(10, 2) self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)])) time.sleep(1) self.assertFalse(limiter.hit(limit)) time.sleep(1) self.assertFalse(limiter.hit(limit))
def test_in_memory_expiry(self): with hiro.Timeline().freeze() as timeline: storage = MemoryStorage() limiter = FixedWindowRateLimiter(storage) per_min = PER_MINUTE(10) for i in range(0, 10): self.assertTrue(limiter.hit(per_min)) timeline.forward(60) # touch another key and yield limiter.hit(PER_SECOND(1)) time.sleep(0.1) self.assertTrue(per_min.key_for() not in storage.storage)
def test_moving_window_redis(self): storage = RedisStorage("redis://localhost:6379") limiter = MovingWindowRateLimiter(storage) limit = PER_SECOND(10, 2) for i in range(0,10): self.assertTrue(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 10 - (i + 1)) time.sleep(2*0.095) self.assertFalse(limiter.hit(limit)) time.sleep(0.4) self.assertTrue(limiter.hit(limit)) self.assertTrue(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 0)
def test_redis(self): storage = RedisStorage("redis://localhost:6379") limiter = FixedWindowRateLimiter(storage) per_min = PER_SECOND(10) start = time.time() count = 0 while time.time() - start < 0.5 and count < 10: self.assertTrue(limiter.hit(per_min)) count += 1 self.assertFalse(limiter.hit(per_min)) while time.time() - start <= 1: time.sleep(0.1) self.assertTrue(limiter.hit(per_min))
def test_in_memory_expiry_moving_window(self): with hiro.Timeline().freeze() as timeline: storage = MemoryStorage() limiter = MovingWindowRateLimiter(storage) per_min = PER_MINUTE(10) per_sec = PER_SECOND(1) for i in range(0, 2): for i in range(0, 10): self.assertTrue(limiter.hit(per_min)) timeline.forward(60) self.assertTrue(limiter.hit(per_sec)) time.sleep(1) self.assertEqual([], storage.events[per_min.key_for()])
def test_memcached(self): storage = MemcachedStorage("memcached://localhost:11211") limiter = FixedWindowRateLimiter(storage) per_min = PER_SECOND(10) start = time.time() count = 0 while time.time() - start < 0.5 and count < 10: self.assertTrue(limiter.hit(per_min)) count += 1 self.assertFalse(limiter.hit(per_min)) while time.time() - start <= 1: time.sleep(0.1) self.assertTrue(limiter.hit(per_min))
def test_fixed_window(self): storage = MemoryStorage() limiter = FixedWindowRateLimiter(storage) with hiro.Timeline().freeze() as timeline: start = int(time.time()) limit = PER_SECOND(10, 2) self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)])) timeline.forward(1) self.assertFalse(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 0) self.assertEqual(limiter.get_window_stats(limit)[0], start + 2) timeline.forward(1) self.assertEqual(limiter.get_window_stats(limit)[1], 10) self.assertTrue(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_in_memory(self): storage = MemoryStorage() limiter = FixedWindowElasticExpiryRateLimiter(storage) with hiro.Timeline().freeze() as timeline: start = int(time.time()) limit = PER_SECOND(10, 2) self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)])) timeline.forward(1) self.assertFalse(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 0) # three extensions to the expiry self.assertEqual(limiter.get_window_stats(limit)[0], start + 3) timeline.forward(1) self.assertFalse(limiter.hit(limit)) timeline.forward(3) start = int(time.time()) self.assertTrue(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 9) self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
def test_large_dataset_redis_moving_window_expiry(self): storage = RedisStorage("redis://localhost:6379") limiter = MovingWindowRateLimiter(storage) limit = PER_SECOND(1000) keys_start = storage.storage.keys('%s/*' % limit.namespace) # 100 routes fake_routes = [uuid4().hex for _ in range(0, 100)] # go as fast as possible in 2 seconds. start = time.time() def smack(e): while not e.is_set(): self.assertTrue(limiter.hit(limit, random.choice(fake_routes))) events = [threading.Event() for _ in range(0, 100)] threads = [threading.Thread(target=smack, args=(e, )) for e in events] [k.start() for k in threads] while time.time() - start < 2: time.sleep(0.1) [k.set() for k in events] time.sleep(2) self.assertTrue(storage.storage.keys("%s/*" % limit.namespace) == [])