def test_can_acquire_from_unknown_domain_url(self): limiter = PerDomainRedisLimiter({'http://globoesporte.globo.com': 10}, redis=self.redis) url = 'http://g1.globo.com/economia/' expect(limiter.acquire(url)).to_be_true() expect(self.redis.zcard('limit-for-http://g1.globo.com')).to_equal(0)
def get_otto_limiter(self): domains = self.cache.get_domain_limiters() limiter = None if domains: limiter = Limiter( *domains, redis=self.redis, expiration_in_seconds=self.config.LIMITER_LOCKS_EXPIRATION) limiter.subscribe_to_lock_miss(self.handle_limiter_miss) return limiter
def get_otto_limiter(self): domains = self.cache.get_domain_limiters() limiter = None if domains: limiter = Limiter( *domains, redis=self.redis, expiration_in_seconds=self.config.LIMITER_LOCKS_EXPIRATION ) limiter.subscribe_to_lock_miss(self.handle_limiter_miss) return limiter
def setUp(self): super(TestPerDomain, self).setUp() self.limiter = PerDomainRedisLimiter( {"http://g1.globo.com": 10}, {"http://globoesporte.globo.com": 10}, redis=self.redis, expiration_in_seconds=12, )
def test_should_call_limiter_miss_twice(self): limiter = PerDomainRedisLimiter( {'http://g1.globo.com': 1}, {'http://globoesporte.globo.com': 1}, redis=self.redis ) limiter.subscribe_to_lock_miss(self.handle_limiter_miss) otto = TornadoOctopus(concurrency=10, auto_start=True, limiter=limiter) otto.enqueue('http://globoesporte.globo.com/', self.handle_url_response) otto.enqueue('http://globoesporte.globo.com/futebol/times/flamengo/', self.handle_url_response) otto.enqueue('http://g1.globo.com/', self.handle_url_response) otto.enqueue('http://g1.globo.com/economia/', self.handle_url_response) otto.wait() expect(self.cache_miss).to_length(2)
def test_should_call_limiter_miss_twice(self): limiter = PerDomainRedisLimiter({'http://g1.globo.com': 1}, {'http://globoesporte.globo.com': 1}, redis=self.redis) limiter.subscribe_to_lock_miss(self.handle_limiter_miss) otto = TornadoOctopus(concurrency=10, auto_start=True, limiter=limiter) otto.enqueue('http://globoesporte.globo.com/', self.handle_url_response) otto.enqueue('http://globoesporte.globo.com/futebol/times/flamengo/', self.handle_url_response) otto.enqueue('http://g1.globo.com/', self.handle_url_response) otto.enqueue('http://g1.globo.com/economia/', self.handle_url_response) otto.wait() expect(self.cache_miss).to_length(2)
def test_can_acquire_from_unknown_domain_url(self): limiter = PerDomainRedisLimiter({"http://globoesporte.globo.com": 10}, redis=self.redis) url = "http://g1.globo.com/economia/" expect(limiter.acquire(url)).to_be_true() expect(self.redis.zcard("limit-for-http://g1.globo.com")).to_equal(0)
class TestPerDomain(TestCase): def setUp(self): super(TestPerDomain, self).setUp() self.limiter = PerDomainRedisLimiter( {"http://g1.globo.com": 10}, {"http://globoesporte.globo.com": 10}, redis=self.redis, expiration_in_seconds=12, ) def test_can_create_limiter(self): expect(self.limiter.redis).to_equal(self.redis) expect(self.limiter.expiration_in_seconds).to_equal(12) expect(self.limiter.domains[0]).to_include("http://g1.globo.com") expect(self.limiter.domains).not_to_be_null() expect(self.limiter.domains[0]["http://g1.globo.com"]).to_equal(10) def test_cant_create_limiter_without_redis(self): try: PerDomainRedisLimiter() except RuntimeError: err = sys.exc_info()[1] expect(err).to_have_an_error_message_of( "You must specify a connection to redis in order to use Redis Limiter." ) else: assert False, "Should not have gotten this far" def test_can_acquire_limit(self): expect(self.limiter.acquire("http://g1.globo.com")).to_be_true() try: expect(self.redis.zcard("limit-for-http://g1.globo.com")).to_equal(1) finally: self.limiter.release("http://g1.globo.com") def test_acquiring_internal_url_gets_proper_domain(self): url = "http://g1.globo.com/economia/" expect(self.limiter.acquire(url)).to_be_true() try: expect(self.redis.zcard("limit-for-http://g1.globo.com")).to_equal(1) finally: self.limiter.release(url) def test_can_acquire_from_unknown_domain_url(self): limiter = PerDomainRedisLimiter({"http://globoesporte.globo.com": 10}, redis=self.redis) url = "http://g1.globo.com/economia/" expect(limiter.acquire(url)).to_be_true() expect(self.redis.zcard("limit-for-http://g1.globo.com")).to_equal(0) def test_can_release(self): url = "http://g1.globo.com/economia/" self.limiter.acquire(url) self.limiter.release(url) expect(self.redis.zcard("limit-for-http://g1.globo.com")).to_equal(0) def test_can_get_domain_from_url(self): expect(self.limiter.get_domain_from_url("http://g1.globo.com/economia/")).to_equal("http://g1.globo.com") def test_can_get_domain_limit(self): url = "http://g1.globo.com/economia/" expect(self.limiter.get_domain_limit(url)).to_equal(10) expect(self.limiter.get_domain_limit("http://www.google.com")).to_equal(0) @patch.object(logging, "info") def test_can_release_unknown_url(self, logging_mock): self.limiter.release("http://www.google.com") expect(self.redis.zcard("limit-for-http://www.google.com")).to_equal(0) logging_mock.assert_called_once_with( "Tried to release lock to a domain that was not specified " "in the limiter (http://www.google.com)." )
def setUp(self): super(TestPerDomain, self).setUp() self.limiter = PerDomainRedisLimiter( {'http://g1.globo.com': 10}, {'http://globoesporte.globo.com': 10}, redis=self.redis, expiration_in_seconds=12)
class TestPerDomain(TestCase): def setUp(self): super(TestPerDomain, self).setUp() self.limiter = PerDomainRedisLimiter( {'http://g1.globo.com': 10}, {'http://globoesporte.globo.com': 10}, redis=self.redis, expiration_in_seconds=12) def test_can_create_limiter(self): expect(self.limiter.redis).to_equal(self.redis) expect(self.limiter.expiration_in_seconds).to_equal(12) expect(self.limiter.domains[0]).to_include('http://g1.globo.com') expect(self.limiter.domains).not_to_be_null() expect(self.limiter.domains[0]['http://g1.globo.com']).to_equal(10) def test_cant_create_limiter_without_redis(self): try: PerDomainRedisLimiter() except RuntimeError: err = sys.exc_info()[1] expect(err).to_have_an_error_message_of( 'You must specify a connection to redis in order to use Redis Limiter.' ) else: assert False, "Should not have gotten this far" def test_can_acquire_limit(self): expect(self.limiter.acquire('http://g1.globo.com')).to_be_true() try: expect( self.redis.zcard('limit-for-http://g1.globo.com')).to_equal(1) finally: self.limiter.release('http://g1.globo.com') def test_acquiring_internal_url_gets_proper_domain(self): url = 'http://g1.globo.com/economia/' expect(self.limiter.acquire(url)).to_be_true() try: expect( self.redis.zcard('limit-for-http://g1.globo.com')).to_equal(1) finally: self.limiter.release(url) def test_can_acquire_from_unknown_domain_url(self): limiter = PerDomainRedisLimiter({'http://globoesporte.globo.com': 10}, redis=self.redis) url = 'http://g1.globo.com/economia/' expect(limiter.acquire(url)).to_be_true() expect(self.redis.zcard('limit-for-http://g1.globo.com')).to_equal(0) def test_can_release(self): url = 'http://g1.globo.com/economia/' self.limiter.acquire(url) self.limiter.release(url) expect(self.redis.zcard('limit-for-http://g1.globo.com')).to_equal(0) def test_can_get_domain_from_url(self): expect( self.limiter.get_domain_from_url('http://g1.globo.com/economia/') ).to_equal('http://g1.globo.com') def test_can_get_domain_limit(self): url = 'http://g1.globo.com/economia/' expect(self.limiter.get_domain_limit(url)).to_equal(10) expect( self.limiter.get_domain_limit('http://www.google.com')).to_equal(0) @patch.object(logging, 'info') def test_can_release_unknown_url(self, logging_mock): self.limiter.release('http://www.google.com') expect(self.redis.zcard('limit-for-http://www.google.com')).to_equal(0) logging_mock.assert_called_once_with( 'Tried to release lock to a domain that was not specified ' 'in the limiter (http://www.google.com).')