def get_otto_limiter(self):
domains = self.cache.get_domain_limiters()
limiter = None
if domains:
limiter = Limiter(
*domains,
redis=self.redis,
expiration_in_seconds=self.config.LIMITER_LOCKS_EXPIRATION)
limiter.subscribe_to_lock_miss(self.handle_limiter_miss)
return limiter
def get_otto_limiter(self):
domains = self.cache.get_domain_limiters()
limiter = None
if domains:
limiter = Limiter(
*domains,
redis=self.redis,
expiration_in_seconds=self.config.LIMITER_LOCKS_EXPIRATION
)
limiter.subscribe_to_lock_miss(self.handle_limiter_miss)
return limiter
def test_should_call_limiter_miss_twice(self):
limiter = PerDomainRedisLimiter(
{'http://g1.globo.com': 1},
{'http://globoesporte.globo.com': 1},
redis=self.redis
)
limiter.subscribe_to_lock_miss(self.handle_limiter_miss)
otto = TornadoOctopus(concurrency=10, auto_start=True, limiter=limiter)
otto.enqueue('http://globoesporte.globo.com/', self.handle_url_response)
otto.enqueue('http://globoesporte.globo.com/futebol/times/flamengo/', self.handle_url_response)
otto.enqueue('http://g1.globo.com/', self.handle_url_response)
otto.enqueue('http://g1.globo.com/economia/', self.handle_url_response)
otto.wait()
expect(self.cache_miss).to_length(2)
def test_should_call_limiter_miss_twice(self):
limiter = PerDomainRedisLimiter({'http://g1.globo.com': 1},
{'http://globoesporte.globo.com': 1},
redis=self.redis)
limiter.subscribe_to_lock_miss(self.handle_limiter_miss)
otto = TornadoOctopus(concurrency=10, auto_start=True, limiter=limiter)
otto.enqueue('http://globoesporte.globo.com/',
self.handle_url_response)
otto.enqueue('http://globoesporte.globo.com/futebol/times/flamengo/',
self.handle_url_response)
otto.enqueue('http://g1.globo.com/', self.handle_url_response)
otto.enqueue('http://g1.globo.com/economia/', self.handle_url_response)
otto.wait()
expect(self.cache_miss).to_length(2)
