def test_can_get_all_limiters(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create(url='http://test.com/')
LimiterFactory.create()
LimiterFactory.create()
limiters = Limiter.get_all(self.db)
expect(limiters).not_to_be_null()
expect(limiters).to_length(3)
expect(limiters).to_include(limiter)
def test_can_convert_user_to_dict(self):
limiter = LimiterFactory.create()
limiter_dict = limiter.to_dict()
expect(limiter_dict['url']).to_equal(limiter.url)
expect(limiter_dict['value']).to_equal(limiter.value)
def test_can_create_limiter(self):
limiter = LimiterFactory.create(url='http://test.com/')
expect(str(limiter)).to_be_like('%s' % limiter.url)
expect(limiter.id).not_to_be_null()
expect(limiter.url).to_equal('http://test.com/')
expect(limiter.value).to_equal(10)
def test_can_get_limiter_by_id(self):
limiter = LimiterFactory.create()
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter.id).to_equal(limiter.id)
invalid_limiter = Limiter.by_id(-1, self.db)
expect(invalid_limiter).to_be_null()
def test_can_get_next_job_when_domain_limited(self):
self.db.query(Domain).delete()
self.db.query(Page).delete()
domain_a = DomainFactory.create()
domain_b = DomainFactory.create()
LimiterFactory.create(url=domain_a.url, value=2)
pages_a = []
pages_b = []
workers = []
for i in range(10):
for j in range(2):
workers.append(WorkerFactory.create())
pages_a.append(PageFactory.create(domain=domain_a, url="%s/%d.html" % (domain_a.url, i), score=i * 10))
pages_b.append(PageFactory.create(domain=domain_b, url="%s/%d.html" % (domain_b.url, i), score=i))
# first one should not be limited
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1,
avg_links_per_page=10
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(pages_a[-1].uuid))
workers[0].current_url = next_job['url']
self.db.flush()
# second one should be limited (2 / 10 = 0.2, rounded up = 1 job at a time)
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(pages_b[-1].uuid))
def test_can_get_limiters(self):
self.db.query(Limiter).delete()
LimiterFactory.create(url='http://test.com/', value=100)
LimiterFactory.create(url='http://globo.com/', value=2)
response = yield self.authenticated_fetch('/limiters')
expect(response.code).to_equal(200)
limiters = loads(response.body)
expect(limiters).to_length(2)
expect(limiters[0]['url']).to_equal('http://test.com/')
expect(limiters[0]['maxValue']).to_equal(100)
expect(limiters[1]['url']).to_equal('http://globo.com/')
expect(limiters[1]['maxValue']).to_equal(2)
def test_can_get_limiters(self):
self.db.query(Limiter).delete()
LimiterFactory.create(url='http://test.com/', value=100)
LimiterFactory.create(url='http://globo.com/', value=2)
response = yield self.authenticated_fetch('/limiters')
expect(response.code).to_equal(200)
limiters = loads(response.body)
expect(limiters).to_length(2)
expect(limiters[0]['url']).to_equal('http://test.com/')
expect(limiters[0]['maxValue']).to_equal(100)
expect(limiters[1]['url']).to_equal('http://globo.com/')
expect(limiters[1]['maxValue']).to_equal(2)
def test_can_get_limiter_by_url(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create(url='http://test.com/')
loaded_limiter = Limiter.by_url('http://test.com/', self.db)
expect(loaded_limiter.id).to_equal(limiter.id)
invalid_limiter = Limiter.by_url('http://test.com/1', self.db)
expect(invalid_limiter).to_be_null()
def test_can_delete_one_limiter(self):
limiter = LimiterFactory.create()
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter.id).to_equal(limiter.id)
affected = Limiter.delete(limiter.id, self.db)
expect(affected).to_equal(1)
deleted_limiter = Limiter.by_id(limiter.id, self.db)
expect(deleted_limiter).to_be_null()
def test_can_get_limiter_by_url_hash(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create(url='http://test.com/')
url_hash = hashlib.sha512('http://test.com/').hexdigest()
loaded_limiter = Limiter.by_url_hash(url_hash, self.db)
expect(loaded_limiter.id).to_equal(limiter.id)
invalid_limiter = Limiter.by_url_hash('00000000', self.db)
expect(invalid_limiter).to_be_null()
def test_can_get_domain_limiters(self):
self.db.query(Limiter).delete()
self.sync_cache.redis.delete('domain-limiters')
domains = self.sync_cache.get_domain_limiters()
expect(domains).to_be_null()
limiter = LimiterFactory.create(url='http://test.com/')
LimiterFactory.create()
LimiterFactory.create()
domains = self.sync_cache.get_domain_limiters()
expect(domains).to_length(3)
expect(domains).to_include({limiter.url: limiter.value})
# should get from cache
self.sync_cache.db = None
domains = self.sync_cache.get_domain_limiters()
expect(domains).to_length(3)
def test_can_get_domain_limiters(self):
self.db.query(Limiter).delete()
self.sync_cache.redis.delete('domain-limiters')
domains = self.sync_cache.get_domain_limiters()
expect(domains).to_be_null()
limiter = LimiterFactory.create(url='http://test.com/')
LimiterFactory.create()
LimiterFactory.create()
domains = self.sync_cache.get_domain_limiters()
expect(domains).to_length(3)
expect(domains).to_include({limiter.url: limiter.value})
# should get from cache
self.sync_cache.db = None
domains = self.sync_cache.get_domain_limiters()
expect(domains).to_length(3)
def test_cant_delete_limiter_as_normal_user(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create()
self.db.query(User).delete()
user = UserFactory(email='*****@*****.**', is_superuser=False)
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter).not_to_be_null()
try:
yield self.authenticated_fetch('/limiters/%d' % limiter.id,
user_email=user.email,
method='DELETE')
except HTTPError, e:
expect(e).not_to_be_null()
expect(e.code).to_equal(401)
expect(e.response.reason).to_be_like('Unauthorized')
def test_cant_delete_limiter_as_normal_user(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create()
self.db.query(User).delete()
user = UserFactory(email='*****@*****.**', is_superuser=False)
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter).not_to_be_null()
try:
yield self.authenticated_fetch(
'/limiters/%d' % limiter.id, user_email=user.email,
method='DELETE'
)
except HTTPError, e:
expect(e).not_to_be_null()
expect(e.code).to_equal(401)
expect(e.response.reason).to_be_like('Unauthorized')
def test_can_delete_limiter_as_superuser(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create()
self.db.query(User).delete()
user = UserFactory(email='*****@*****.**', is_superuser=True)
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter).not_to_be_null()
response = yield self.authenticated_fetch('/limiters/%d' % limiter.id,
user_email=user.email,
method='DELETE')
expect(response.code).to_equal(204)
expect(response.body).to_length(0)
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter).to_be_null()
def test_can_delete_limiter_as_superuser(self):
self.db.query(Limiter).delete()
limiter = LimiterFactory.create()
self.db.query(User).delete()
user = UserFactory(email='*****@*****.**', is_superuser=True)
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter).not_to_be_null()
response = yield self.authenticated_fetch(
'/limiters/%d' % limiter.id, user_email=user.email,
method='DELETE'
)
expect(response.code).to_equal(204)
expect(response.body).to_length(0)
loaded_limiter = Limiter.by_id(limiter.id, self.db)
expect(loaded_limiter).to_be_null()
def test_limiter_matches_url(self):
limiter = LimiterFactory.create(url='http://test.com/')
expect(limiter.matches('http://test.com/1.html')).to_be_true()
expect(limiter.matches('http://test2.com/1.html')).to_be_false()
