def test_can_get_all_limiters(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create(url='http://test.com/') LimiterFactory.create() LimiterFactory.create() limiters = Limiter.get_all(self.db) expect(limiters).not_to_be_null() expect(limiters).to_length(3) expect(limiters).to_include(limiter)
def test_can_convert_user_to_dict(self): limiter = LimiterFactory.create() limiter_dict = limiter.to_dict() expect(limiter_dict['url']).to_equal(limiter.url) expect(limiter_dict['value']).to_equal(limiter.value)
def test_can_create_limiter(self): limiter = LimiterFactory.create(url='http://test.com/') expect(str(limiter)).to_be_like('%s' % limiter.url) expect(limiter.id).not_to_be_null() expect(limiter.url).to_equal('http://test.com/') expect(limiter.value).to_equal(10)
def test_can_get_limiter_by_id(self): limiter = LimiterFactory.create() loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter.id).to_equal(limiter.id) invalid_limiter = Limiter.by_id(-1, self.db) expect(invalid_limiter).to_be_null()
def test_can_get_next_job_when_domain_limited(self): self.db.query(Domain).delete() self.db.query(Page).delete() domain_a = DomainFactory.create() domain_b = DomainFactory.create() LimiterFactory.create(url=domain_a.url, value=2) pages_a = [] pages_b = [] workers = [] for i in range(10): for j in range(2): workers.append(WorkerFactory.create()) pages_a.append(PageFactory.create(domain=domain_a, url="%s/%d.html" % (domain_a.url, i), score=i * 10)) pages_b.append(PageFactory.create(domain=domain_b, url="%s/%d.html" % (domain_b.url, i), score=i)) # first one should not be limited next_job = Page.get_next_job( self.db, expiration=100, cache=self.sync_cache, lock_expiration=1, avg_links_per_page=10 ) expect(next_job).not_to_be_null() expect(next_job['page']).to_equal(str(pages_a[-1].uuid)) workers[0].current_url = next_job['url'] self.db.flush() # second one should be limited (2 / 10 = 0.2, rounded up = 1 job at a time) next_job = Page.get_next_job( self.db, expiration=100, cache=self.sync_cache, lock_expiration=1 ) expect(next_job).not_to_be_null() expect(next_job['page']).to_equal(str(pages_b[-1].uuid))
def test_can_get_limiters(self): self.db.query(Limiter).delete() LimiterFactory.create(url='http://test.com/', value=100) LimiterFactory.create(url='http://globo.com/', value=2) response = yield self.authenticated_fetch('/limiters') expect(response.code).to_equal(200) limiters = loads(response.body) expect(limiters).to_length(2) expect(limiters[0]['url']).to_equal('http://test.com/') expect(limiters[0]['maxValue']).to_equal(100) expect(limiters[1]['url']).to_equal('http://globo.com/') expect(limiters[1]['maxValue']).to_equal(2)
def test_can_get_limiter_by_url(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create(url='http://test.com/') loaded_limiter = Limiter.by_url('http://test.com/', self.db) expect(loaded_limiter.id).to_equal(limiter.id) invalid_limiter = Limiter.by_url('http://test.com/1', self.db) expect(invalid_limiter).to_be_null()
def test_can_delete_one_limiter(self): limiter = LimiterFactory.create() loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter.id).to_equal(limiter.id) affected = Limiter.delete(limiter.id, self.db) expect(affected).to_equal(1) deleted_limiter = Limiter.by_id(limiter.id, self.db) expect(deleted_limiter).to_be_null()
def test_can_get_limiter_by_url_hash(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create(url='http://test.com/') url_hash = hashlib.sha512('http://test.com/').hexdigest() loaded_limiter = Limiter.by_url_hash(url_hash, self.db) expect(loaded_limiter.id).to_equal(limiter.id) invalid_limiter = Limiter.by_url_hash('00000000', self.db) expect(invalid_limiter).to_be_null()
def test_can_get_domain_limiters(self): self.db.query(Limiter).delete() self.sync_cache.redis.delete('domain-limiters') domains = self.sync_cache.get_domain_limiters() expect(domains).to_be_null() limiter = LimiterFactory.create(url='http://test.com/') LimiterFactory.create() LimiterFactory.create() domains = self.sync_cache.get_domain_limiters() expect(domains).to_length(3) expect(domains).to_include({limiter.url: limiter.value}) # should get from cache self.sync_cache.db = None domains = self.sync_cache.get_domain_limiters() expect(domains).to_length(3)
def test_cant_delete_limiter_as_normal_user(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create() self.db.query(User).delete() user = UserFactory(email='*****@*****.**', is_superuser=False) loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter).not_to_be_null() try: yield self.authenticated_fetch('/limiters/%d' % limiter.id, user_email=user.email, method='DELETE') except HTTPError, e: expect(e).not_to_be_null() expect(e.code).to_equal(401) expect(e.response.reason).to_be_like('Unauthorized')
def test_cant_delete_limiter_as_normal_user(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create() self.db.query(User).delete() user = UserFactory(email='*****@*****.**', is_superuser=False) loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter).not_to_be_null() try: yield self.authenticated_fetch( '/limiters/%d' % limiter.id, user_email=user.email, method='DELETE' ) except HTTPError, e: expect(e).not_to_be_null() expect(e.code).to_equal(401) expect(e.response.reason).to_be_like('Unauthorized')
def test_can_delete_limiter_as_superuser(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create() self.db.query(User).delete() user = UserFactory(email='*****@*****.**', is_superuser=True) loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter).not_to_be_null() response = yield self.authenticated_fetch('/limiters/%d' % limiter.id, user_email=user.email, method='DELETE') expect(response.code).to_equal(204) expect(response.body).to_length(0) loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter).to_be_null()
def test_can_delete_limiter_as_superuser(self): self.db.query(Limiter).delete() limiter = LimiterFactory.create() self.db.query(User).delete() user = UserFactory(email='*****@*****.**', is_superuser=True) loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter).not_to_be_null() response = yield self.authenticated_fetch( '/limiters/%d' % limiter.id, user_email=user.email, method='DELETE' ) expect(response.code).to_equal(204) expect(response.body).to_length(0) loaded_limiter = Limiter.by_id(limiter.id, self.db) expect(loaded_limiter).to_be_null()
def test_limiter_matches_url(self): limiter = LimiterFactory.create(url='http://test.com/') expect(limiter.matches('http://test.com/1.html')).to_be_true() expect(limiter.matches('http://test2.com/1.html')).to_be_false()