def test_failed_redis_lock(self):
storage = RedisStorage("redis://localhost:6379")
limiter = MovingWindowRateLimiter(storage)
limit = RateLimitItemPerSecond(1000)
key = limit.key_for("test") + "/LOCK"
storage.storage.setnx(key, 1)
self.assertRaises(redis.lock.LockError, limiter.hit, limit, "test")
def test_failed_redis_sentinel_lock(self):
storage = RedisSentinelStorage("redis+sentinel://localhost:26379", service_name="localhost-redis-sentinel")
limiter = MovingWindowRateLimiter(storage)
limit = RateLimitItemPerSecond(1000)
key = limit.key_for("test") + "/LOCK"
storage.sentinel.master_for("localhost-redis-sentinel").setnx(key, 1)
self.assertRaises(redis.lock.LockError, limiter.hit, limit, "test")
def test_fixed_window_with_elastic_expiry_memcache_concurrency(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
start = int(time.time())
limit = RateLimitItemPerSecond(100, 2)
def _c():
for i in range(0,50):
limiter.hit(limit)
t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c)
t1.start(), t2.start()
[t1.join(), t2.join()]
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
self.assertTrue(start + 2 <= limiter.get_window_stats(limit)[0] <= start + 3)
self.assertEqual(storage.get(limit.key_for()), 100)
def test_redis(self):
storage = RedisStorage("redis://localhost:6379")
limiter = FixedWindowRateLimiter(storage)
per_min = RateLimitItemPerSecond(10)
start = time.time()
count = 0
while time.time() - start < 0.5 and count < 10:
self.assertTrue(limiter.hit(per_min))
count += 1
self.assertFalse(limiter.hit(per_min))
while time.time() - start <= 1:
time.sleep(0.1)
self.assertTrue(limiter.hit(per_min))
def test_moving_window_redis(self):
storage = RedisStorage("redis://localhost:6379")
limiter = MovingWindowRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
for i in range(0, 10):
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 10 - (i + 1))
time.sleep(2 * 0.095)
self.assertFalse(limiter.hit(limit))
time.sleep(0.4)
self.assertTrue(limiter.hit(limit))
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
def test_in_memory_expiry_moving_window(self):
with hiro.Timeline().freeze() as timeline:
storage = MemoryStorage()
limiter = MovingWindowRateLimiter(storage)
per_min = RateLimitItemPerMinute(10)
per_sec = RateLimitItemPerSecond(1)
for i in range(0,2):
for i in range(0,10):
self.assertTrue(limiter.hit(per_min))
timeline.forward(60)
self.assertTrue(limiter.hit(per_sec))
time.sleep(1)
self.assertEqual([], storage.events[per_min.key_for()])
def test_memcached(self):
storage = MemcachedStorage("memcached://localhost:11211")
limiter = FixedWindowRateLimiter(storage)
per_min = RateLimitItemPerSecond(10)
start = time.time()
count = 0
while time.time() - start < 0.5 and count < 10:
self.assertTrue(limiter.hit(per_min))
count += 1
self.assertFalse(limiter.hit(per_min))
while time.time() - start <= 1:
time.sleep(0.1)
self.assertTrue(limiter.hit(per_min))
def test_redis_sentinel(self):
storage = RedisSentinelStorage("redis+sentinel://localhost:26379",
service_name="localhost-redis-sentinel")
limiter = FixedWindowRateLimiter(storage)
per_min = RateLimitItemPerSecond(10)
start = time.time()
count = 0
while time.time() - start < 0.5 and count < 10:
self.assertTrue(limiter.hit(per_min))
count += 1
self.assertFalse(limiter.hit(per_min))
while time.time() - start <= 1:
time.sleep(0.1)
self.assertTrue(limiter.hit(per_min))
def test_fixed_window(self):
storage = MemoryStorage()
limiter = FixedWindowRateLimiter(storage)
with hiro.Timeline().freeze() as timeline:
start = int(time.time())
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)]))
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
timeline.forward(1)
self.assertEqual(limiter.get_window_stats(limit)[1], 10)
self.assertTrue(limiter.hit(limit))
def test_moving_window_expiry(self):
limiter = MovingWindowRateLimiter(self.storage)
limit = RateLimitItemPerSecond(2)
self.assertTrue(limiter.hit(limit))
time.sleep(0.9)
self.assertTrue(limiter.hit(limit))
self.assertFalse(limiter.hit(limit))
time.sleep(0.1)
self.assertTrue(limiter.hit(limit))
last = time.time()
while time.time() - last <= 1:
time.sleep(0.05)
self.assertTrue(
self.storage.storage.keys("%s/*" % limit.namespace) == [])
def test_fixed_window_with_elastic_expiry_in_memory(self):
storage = MemoryStorage()
limiter = FixedWindowElasticExpiryRateLimiter(storage)
with hiro.Timeline().freeze() as timeline:
start = int(time.time())
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)]))
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
# three extensions to the expiry
self.assertEqual(limiter.get_window_stats(limit)[0], start + 3)
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
timeline.forward(3)
start = int(time.time())
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 9)
self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
def test_large_dataset_redis_moving_window_expiry(self):
storage = RedisStorage("redis://localhost:6379")
limiter = MovingWindowRateLimiter(storage)
limit = RateLimitItemPerSecond(1000)
keys_start = storage.storage.keys('%s/*' % limit.namespace)
# 100 routes
fake_routes = [uuid4().hex for _ in range(0,100)]
# go as fast as possible in 2 seconds.
start = time.time()
def smack(e):
while not e.is_set():
self.assertTrue(limiter.hit(limit, random.choice(fake_routes)))
events = [threading.Event() for _ in range(0,100)]
threads = [threading.Thread(target=smack, args=(e,)) for e in events]
[k.start() for k in threads]
while time.time() - start < 2:
time.sleep(0.1)
[k.set() for k in events]
time.sleep(2)
self.assertTrue(storage.storage.keys("%s/*" % limit.namespace) == [])
def test_memory_storage_moving_window(self):
storage = MemoryStorage()
limiter = MovingWindowRateLimiter(storage)
per_second = RateLimitItemPerSecond(100)
[limiter.hit(per_second, uuid4().hex) for _ in range(100)]
key = uuid4().hex
hits = []
def hit():
if limiter.hit(per_second, key):
hits.append(None)
start = time.time()
threads = [threading.Thread(target=hit) for _ in range(1000)]
[t.start() for t in threads]
[t.join() for t in threads]
self.assertTrue(time.time() - start < 1)
self.assertEqual(len(hits), 100)
