def test_increment_active_review_count(self): key = 'g.com-active-review-count' self.cache.redis.delete(key) gcom = DomainFactory.create(url='http://g.com', name='g.com') page = PageFactory.create(domain=gcom) ReviewFactory.create( is_active=True, is_complete=True, domain=gcom, page=page, number_of_violations=1 ) page = PageFactory.create(domain=gcom) ReviewFactory.create( is_active=False, is_complete=True, domain=gcom, page=page, number_of_violations=3 ) page_count = yield self.cache.get_active_review_count('g.com') expect(page_count).to_equal(1) yield self.cache.increment_active_review_count('g.com') page_count = yield self.cache.get_active_review_count('g.com') expect(page_count).to_equal(2)
def test_can_get_page_reviews(self): dt1 = datetime(2010, 11, 12, 13, 14, 15) dt1_timestamp = calendar.timegm(dt1.utctimetuple()) dt2 = datetime(2011, 12, 13, 14, 15, 16) dt2_timestamp = calendar.timegm(dt2.utctimetuple()) domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain) review1 = ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt1, number_of_violations=20) review2 = ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=30) response = yield self.http_client.fetch( self.get_url('/page/%s/reviews/' % page.uuid) ) expect(response.code).to_equal(200) page_details = loads(response.body) expect(page_details[0]['violationCount']).to_equal(30) expect(page_details[0]['uuid']).to_equal(str(review2.uuid)) expect(page_details[0]['completedAt']).to_equal(dt2_timestamp) expect(page_details[1]['violationCount']).to_equal(20) expect(page_details[1]['uuid']).to_equal(str(review1.uuid)) expect(page_details[1]['completedAt']).to_equal(dt1_timestamp)
def test_can_get_domain_details(self): domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain, url=domain.url) page2 = PageFactory.create(domain=domain) ReviewFactory.create(page=page, is_active=True, is_complete=True, number_of_violations=20) ReviewFactory.create(page=page2, is_active=True, is_complete=True, number_of_violations=30) response = yield self.authenticated_fetch('/domains/%s/' % domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['name']).to_equal('domain-details.com') expect(domain_details['pageCount']).to_equal(2) expect(domain_details['reviewCount']).to_equal(2) expect(domain_details['violationCount']).to_equal(50) expect(domain_details['reviewPercentage']).to_equal(100.00) expect(domain_details['errorPercentage']).to_equal(0) expect(domain_details['averageResponseTime']).to_equal(0) expect(domain_details['is_active']).to_be_true() expect(domain_details['homepageId']).to_equal(str(page.uuid)) expect(domain_details['homepageReviewId']).to_equal(str(page.last_review_uuid))
def test_increment_active_review_count(self): key = 'g.com-active-review-count' self.sync_cache.redis.delete(key) gcom = DomainFactory.create(url='http://g.com', name='g.com') page = PageFactory.create(domain=gcom) ReviewFactory.create( is_active=True, is_complete=True, domain=gcom, page=page, number_of_violations=1 ) page = PageFactory.create(domain=gcom) ReviewFactory.create( is_active=False, is_complete=True, domain=gcom, page=page, number_of_violations=3 ) self.sync_cache.increment_active_review_count(gcom.name) active_review_count = self.sync_cache.redis.get(key) expect(active_review_count).to_equal('1') self.sync_cache.increment_active_review_count(gcom.name) active_review_count = self.sync_cache.redis.get(key) expect(active_review_count).to_equal('2')
def test_can_get_domain_details(self): domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain, url=domain.url) page2 = PageFactory.create(domain=domain) ReviewFactory.create(page=page, is_active=True, is_complete=True, number_of_violations=20) ReviewFactory.create(page=page2, is_active=True, is_complete=True, number_of_violations=30) response = yield self.authenticated_fetch('/domains/%s/' % domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['name']).to_equal('domain-details.com') expect(domain_details['pageCount']).to_equal(2) expect(domain_details['reviewCount']).to_equal(2) expect(domain_details['violationCount']).to_equal(50) expect(domain_details['reviewPercentage']).to_equal(100.00) expect(domain_details['errorPercentage']).to_equal(0) expect(domain_details['averageResponseTime']).to_equal(0) expect(domain_details['is_active']).to_be_true() expect(domain_details['homepageId']).to_equal(str(page.uuid)) expect(domain_details['homepageReviewId']).to_equal( str(page.last_review_uuid))
def test_can_get_reviews_for_domain(self): dt = datetime(2013, 10, 10, 10, 10, 10) dt2 = datetime(2013, 10, 11, 10, 10, 10) dt3 = datetime(2013, 10, 12, 10, 10, 10) domain = DomainFactory.create() page = PageFactory.create(domain=domain, last_review_date=dt3) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt, number_of_violations=20) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=10) review = ReviewFactory.create(page=page, is_active=True, is_complete=True, completed_date=dt3, number_of_violations=30) reviews = domain.get_active_reviews(self.db) expect(reviews).to_length(1) expect(reviews[0].last_review_uuid).to_equal(str(review.uuid))
def test_can_get_domain_details(self): self.clean_cache('domain-details.com') domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain) page2 = PageFactory.create(domain=domain) ReviewFactory.create(page=page, is_active=True, is_complete=True, number_of_violations=20) ReviewFactory.create(page=page2, is_active=True, is_complete=True, number_of_violations=30) response = yield self.http_client.fetch( self.get_url('/domains/%s/' % domain.name) ) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['name']).to_equal('domain-details.com') expect(domain_details['pageCount']).to_equal(2) expect(domain_details['reviewCount']).to_equal(2) expect(domain_details['violationCount']).to_equal(50) expect(domain_details['reviewPercentage']).to_equal(100.00) expect(domain_details['statusCodeInfo']).to_equal([]) expect(domain_details['errorPercentage']).to_equal(0) expect(domain_details['averageResponseTime']).to_equal(0) expect(domain_details['is_active']).to_be_true()
def test_can_get_page_reviews(self): dt1 = datetime(2010, 11, 12, 13, 14, 15) dt1_timestamp = calendar.timegm(dt1.utctimetuple()) dt2 = datetime(2011, 12, 13, 14, 15, 16) dt2_timestamp = calendar.timegm(dt2.utctimetuple()) domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain) review1 = ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt1, number_of_violations=20) review2 = ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=30) response = yield self.authenticated_fetch( '/page/%s/reviews/' % page.uuid ) expect(response.code).to_equal(200) page_details = loads(response.body) expect(page_details[0]['violationCount']).to_equal(30) expect(page_details[0]['uuid']).to_equal(str(review2.uuid)) expect(page_details[0]['completedAt']).to_equal(dt2_timestamp) expect(page_details[1]['violationCount']).to_equal(20) expect(page_details[1]['uuid']).to_equal(str(review1.uuid)) expect(page_details[1]['completedAt']).to_equal(dt1_timestamp)
def test_can_get_last_reviews_count_in_last_hour_filter_by_domain(self): dt = datetime.utcnow() domain1 = DomainFactory.create() domain2 = DomainFactory.create() page1 = PageFactory.create(domain=domain1) page2 = PageFactory.create(domain=domain2) ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=1), page=page1) ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=59), page=page1) ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=1), page=page2) ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=59), page=page2) self.db.flush() response = yield self.authenticated_fetch( '/reviews-in-last-hour?domain_filter=%s' % domain1.name) expect(response.code).to_equal(200) result = loads(response.body) expect(result['count']).to_equal(2)
def test_can_get_last_reviews_count_in_last_hour(self, datetime_mock): dt = datetime(2014, 2, 14, 15, 0, 30) datetime_mock.utcnow.return_value = dt ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=1)) first_date = dt - timedelta(minutes=59) ReviewFactory.create(is_active=True, completed_date=first_date) ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=5)) ReviewFactory.create(is_active=True, completed_date=dt - timedelta(minutes=61)) self.db.flush() self.authenticated_fetch('/reviews-in-last-hour', callback=self.stop) response = self.wait() expect(response.code).to_equal(200) result = loads(response.body) expect(result['count']).to_equal(3) expect(round(result['ellapsed'], 0)).to_be_like(59 * 60)
def test_can_get_last_reviews_count_in_last_hour(self): dt = datetime.utcnow() ReviewFactory.create( is_active=True, completed_date=dt - timedelta(minutes=1) ) first_date = dt - timedelta(minutes=59) ReviewFactory.create( is_active=True, completed_date=first_date ) ReviewFactory.create( is_active=True, completed_date=dt - timedelta(minutes=5) ) ReviewFactory.create( is_active=True, completed_date=dt - timedelta(minutes=61) ) self.db.flush() url = self.get_url('/reviews-in-last-hour') response = yield self.http_client.fetch(url, method='GET') expect(response.code).to_equal(200) result = loads(response.body) expect(result['count']).to_equal(3) expect(round(result['ellapsed'], 0)).to_be_like(59.0 * 60)
def test_can_get_domain_reviews(self): dt = datetime(2010, 11, 12, 13, 14, 15) dt_timestamp = calendar.timegm(dt.utctimetuple()) dt2 = datetime(2011, 12, 13, 14, 15, 16) dt2_timestamp = calendar.timegm(dt2.utctimetuple()) domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain, last_review_date=dt) page2 = PageFactory.create(domain=domain, last_review_date=dt2) ReviewFactory.create(page=page, is_active=True, is_complete=True, completed_date=dt, number_of_violations=20) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=30) ReviewFactory.create(page=page2, is_active=True, is_complete=True, completed_date=dt2, number_of_violations=30) ReviewFactory.create(page=page2, is_active=False, is_complete=True, completed_date=dt, number_of_violations=20) response = yield self.http_client.fetch( self.get_url('/domains/%s/reviews/' % domain.name) ) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['pages']).to_length(2) expect(domain_details['pages'][1]['url']).to_equal(page2.url) expect(domain_details['pages'][1]['uuid']).to_equal(str(page2.uuid)) expect(domain_details['pages'][1]['completedAt']).to_equal(dt2_timestamp) expect(domain_details['pages'][0]['url']).to_equal(page.url) expect(domain_details['pages'][0]['uuid']).to_equal(str(page.uuid)) expect(domain_details['pages'][0]['completedAt']).to_equal(dt_timestamp)
def test_can_get_last_reviews_count_in_last_hour(self, datetime_mock): dt = datetime(2014, 2, 14, 15, 0, 30) datetime_mock.utcnow.return_value = dt ReviewFactory.create( is_active=True, completed_date=dt - timedelta(minutes=1) ) first_date = dt - timedelta(minutes=59) ReviewFactory.create( is_active=True, completed_date=first_date ) ReviewFactory.create( is_active=True, completed_date=dt - timedelta(minutes=5) ) ReviewFactory.create( is_active=True, completed_date=dt - timedelta(minutes=61) ) self.db.flush() self.authenticated_fetch('/reviews-in-last-hour', callback=self.stop) response = self.wait() expect(response.code).to_equal(200) result = loads(response.body) expect(result['count']).to_equal(3) expect(round(result['ellapsed'], 0)).to_be_like(59 * 60)
def test_can_search(self): dt = datetime.now() page = PageFactory.create(url="http://www.mypage.something.com") review1 = ReviewFactory.create( page=page, is_active=True, is_complete=True, completed_date=dt, number_of_violations=20 ) self.db.flush() page.last_review = review1 page.last_review_date = dt self.db.flush() response = yield self.http_client.fetch( self.get_url('/search?term=http://www.mypage.something.com'), method='GET', ) expect(response.code).to_equal(200) obj = loads(response.body) expect(obj).to_be_like({ u'url': u'http://www.mypage.something.com', u'reviewId': str(review1.uuid), u'uuid': str(page.uuid) })
def test_can_search(self): dt = datetime.now() page = PageFactory.create(url="http://www.mypage.something.com") review1 = ReviewFactory.create(page=page, is_active=True, is_complete=True, completed_date=dt, number_of_violations=20) self.db.flush() page.last_review = review1 page.last_review_date = dt self.db.flush() response = yield self.authenticated_fetch( '/search?term=http://www.mypage.something.com') expect(response.code).to_equal(200) obj = loads(response.body) expect(obj).to_be_like({ u'url': u'http://www.mypage.something.com', u'reviewId': str(review1.uuid), u'uuid': str(page.uuid), u'domain': 'mypage.something.com' })
def test_can_get_violation_count_for_domain(self): self.db.query(Domain).delete() globocom = DomainFactory.create(url="http://globo.com", name="globo.com") page = PageFactory.create(domain=globocom) ReviewFactory.create(is_active=True, is_complete=True, domain=globocom, page=page, number_of_violations=10) violation_count = yield self.cache.get_violation_count('globo.com') expect(violation_count).to_equal(10) # should get from cache self.cache.db = None violation_count = yield self.cache.get_violation_count('globo.com') expect(violation_count).to_equal(10)
def test_can_get_domains_full_data(self): domains = [] for i in range(3): domains.append(DomainFactory.create(name='domain-%d.com' % i)) pages = [] for i, domain in enumerate(domains): pages.append([]) for j in range(3): pages[i].append(PageFactory.create(domain=domain)) requests = reviews = [] for i, (domain, page) in enumerate(zip(domains, pages)): for j in range(i + 1): reviews.append( ReviewFactory.create(domain=domain, page=page[j], is_active=True, number_of_violations=(5 + 2 * j))) requests.append( RequestFactory.create(status_code=200 if j % 2 == 0 else 404, domain_name=domain.name, response_time=0.25 * (i + 1))) self.server.application.violation_definitions = { 'key.%s' % i: { 'title': 'title.%s' % i, 'category': 'category.%s' % (i % 3), 'key': Key.get_or_create(self.db, 'key.%d' % i, 'category.%d' % (i % 3)) } for i in range(9) } response = yield self.authenticated_fetch('/domains-details') expect(response.code).to_equal(200) full_data = loads(response.body) expect(full_data).to_length(3) expect(full_data[0].keys()).to_length(10) expect(map(lambda d: d['name'], full_data)).to_be_like( ['domain-0.com', 'domain-1.com', 'domain-2.com']) expect(map(lambda d: d['pageCount'], full_data)).to_be_like([3, 3, 3]) expect(map(lambda d: d['reviewCount'], full_data)).to_be_like([1, 2, 3]) expect(map(lambda d: d['violationCount'], full_data)).to_be_like([5, 12, 21]) expect(map(lambda d: d['reviewPercentage'], full_data)).to_be_like([33.33, 66.67, 100.0]) expect(map(lambda d: d['errorPercentage'], full_data)).to_be_like([0.0, 50.0, 33.33]) expect(map(lambda d: d['averageResponseTime'], full_data)).to_be_like([0.25, 0.5, 0.75])
def test_to_dict(self): page_id = uuid4() review_id = uuid4() page = PageFactory.build(uuid=page_id) review = ReviewFactory.build(page=page, uuid=review_id) key1 = KeyFactory.create(name='some.random.key1') review.add_violation(key1, 'b', 100, review.domain) key2 = KeyFactory.create(name='some.random.key2') review.add_fact(key2, 'b') fact_definitions = {'some.random.key2': {}} violation_definitions = {'some.random.key1': {}} expect(review.to_dict(fact_definitions, violation_definitions)).to_be_like({ 'domain': review.domain.name, 'uuid': str(review_id), 'completedAt': None, 'facts': [ {'value': 'b', 'key': 'some.random.key2', 'unit': 'value', 'title': 'unknown', 'category': 'unknown'} ], 'violations': [ {'points': 100, 'key': 'some.random.key1', 'description': 'b', 'title': 'undefined', 'category': 'undefined'} ], 'page': { 'url': page.url, 'lastModified': None, 'expires': None, 'uuid': str(page_id), 'score': 0.0 }, 'createdAt': review.created_date, 'isComplete': False })
def test_can_gen_doc(self): review = ReviewFactory.create( is_active=True, number_of_violations=6, created_date=datetime(2014, 04, 15, 11, 44), completed_date=datetime(2014, 04, 15, 11, 44), ) expected = { 'keys': [{ 'id': violation.key_id } for violation in review.violations], 'uuid': str(review.uuid), 'completed_date': review.completed_date, 'violation_count': 6, 'page_id': review.page_id, 'page_uuid': str(review.page.uuid), 'page_url': review.page.url, 'page_last_review_date': review.page.last_review_date, 'domain_id': review.domain_id, 'domain_name': review.domain.name, } doc = self.ES.gen_doc(review) expect(doc).to_be_like(expected)
def test_worker_is_working(self): review = ReviewFactory.create() worker = WorkerFactory.create() worker2 = WorkerFactory.create(current_url=review.domain.url) expect(worker.working).to_be_false() expect(worker2.working).to_be_true()
def test_can_get_review_by_uuid(self): review = ReviewFactory.create() loaded = Review.by_uuid(review.uuid, self.db) expect(loaded.id).to_equal(review.id) invalid = Review.by_uuid(uuid4(), self.db) expect(invalid).to_be_null()
def test_worker_to_dict(self): review = ReviewFactory.create() worker = WorkerFactory.create(current_url=review.domain.url) worker_dict = worker.to_dict() expect(worker_dict['uuid']).to_equal(str(worker.uuid)) expect(worker_dict['last_ping']).to_equal(str(worker.last_ping)) expect(worker_dict['working']).to_be_true()
def test_can_get_last_reviews(self): page = PageFactory.create() date_now = datetime(2013, 11, 12, 13, 25, 27) review = ReviewFactory.create(page=page, is_active=True, is_complete=False, completed_date=date_now, created_date=date_now) key1 = KeyFactory.create(name='fact') review.add_fact(key1, 'value') key2 = KeyFactory.create(name='violation') review.add_violation(key2, 'value', 100, page.domain) review.is_complete = True self.db.flush() response = yield self.authenticated_fetch('/last-reviews') expect(response.code).to_equal(200) dt = calendar.timegm(date_now.utctimetuple()) expected = [{ 'domain': review.domain.name, 'page': page.to_dict(), 'uuid': str(review.uuid), 'isComplete': True, 'facts': [{ u'key': u'fact', u'unit': u'value', u'value': u'value', u'title': u'unknown', u'category': u'unknown' }], 'violations': [{ u'points': 100, u'description': u'value', u'key': u'violation', u'title': u'undefined', u'category': 'undefined' }], 'createdAt': dt, 'completedAt': dt, 'violationCount': 1, }] expect(loads(response.body)).to_be_like(expected)
def test_can_append_facts(self): review = ReviewFactory.build() expect(review.facts).to_length(0) key = KeyFactory.create(name='a') review.add_fact(key, 'b') expect(review.facts).to_length(1) expect(review.facts[0].key.name).to_equal('a') expect(review.facts[0].value).to_equal('b')
def test_invalid_review_uuid_returns_redirect(self): page = PageFactory.create() review = ReviewFactory.create(page=page) response = yield self.authenticated_fetch('/page/%s/review/%s' % (page.uuid, self.ZERO_UUID)) expect(response.code).to_equal(200) expect(str(review.uuid)).to_equal(loads(response.body).get('uuid'))
def test_can_get_violations_per_domain(self): domain = DomainFactory.create() domain2 = DomainFactory.create() DomainFactory.create() page = PageFactory.create(domain=domain) page2 = PageFactory.create(domain=domain) page3 = PageFactory.create(domain=domain2) page4 = PageFactory.create(domain=domain2) page5 = PageFactory.create(domain=domain2) ReviewFactory.create(domain=domain, page=page, number_of_violations=40, is_active=True, is_complete=True) ReviewFactory.create(domain=domain, page=page2, number_of_violations=20, is_active=True, is_complete=True) ReviewFactory.create(domain=domain2, page=page3, number_of_violations=15, is_active=True, is_complete=True) ReviewFactory.create(domain=domain2, page=page4, number_of_violations=25, is_active=True, is_complete=True) ReviewFactory.create(domain=domain2, page=page5, number_of_violations=50, is_active=True, is_complete=True) violations_per_domain = Domain.get_violations_per_domain(self.db) expect(violations_per_domain).to_be_like({ domain.id: 60, domain2.id: 90 })
def test_can_get_domain_reviews_using_elastic_search_provider(self): self.use_elastic_search_provider() dt = datetime(2010, 11, 12, 13, 14, 15) dt_timestamp = calendar.timegm(dt.utctimetuple()) dt2 = datetime(2011, 12, 13, 14, 15, 16) dt2_timestamp = calendar.timegm(dt2.utctimetuple()) domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") page = PageFactory.create(domain=domain, last_review_date=dt) page2 = PageFactory.create(domain=domain, last_review_date=dt2) review = ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt, number_of_violations=13) self.server.application.search_provider.index_review(review) review = ReviewFactory.create(page=page, is_active=True, is_complete=True, completed_date=dt, number_of_violations=12) self.server.application.search_provider.index_review(review) review = ReviewFactory.create(page=page2, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=11) self.server.application.search_provider.index_review(review) review = ReviewFactory.create(page=page2, is_active=True, is_complete=True, completed_date=dt2, number_of_violations=10) self.server.application.search_provider.index_review(review) self.server.application.search_provider.refresh() response = yield self.authenticated_fetch('/domains/%s/reviews/' % domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['pages']).to_length(2) expect(domain_details['pages'][0]['url']).to_equal(page.url) expect(domain_details['pages'][0]['uuid']).to_equal(str(page.uuid)) expect(domain_details['pages'][0]['completedAt']).to_equal(dt_timestamp) expect(domain_details['pages'][0]['violationCount']).to_equal(12) expect(domain_details['pages'][1]['url']).to_equal(page2.url) expect(domain_details['pages'][1]['uuid']).to_equal(str(page2.uuid)) expect(domain_details['pages'][1]['completedAt']).to_equal(dt2_timestamp) expect(domain_details['pages'][1]['violationCount']).to_equal(10)
def test_increment_violations_count(self): key = 'g.com-violation-count' self.cache.redis.delete(key) gcom = DomainFactory.create(url='http://g.com', name='g.com') page = PageFactory.create(domain=gcom) ReviewFactory.create( is_active=True, is_complete=True, domain=gcom, page=page, number_of_violations=10 ) violation_count = yield self.cache.get_violation_count(gcom.name) expect(violation_count).to_equal(10) yield self.cache.increment_violations_count(gcom.name) violation_count = yield self.cache.get_violation_count(gcom.name) expect(violation_count).to_equal(11)
def test_can_get_reviews_for_domain(self): dt = datetime(2013, 10, 10, 10, 10, 10) dt2 = datetime(2013, 10, 11, 10, 10, 10) dt3 = datetime(2013, 10, 12, 10, 10, 10) domain = DomainFactory.create() page = PageFactory.create(domain=domain, last_review_date=dt3) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt, number_of_violations=20) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=10) review = ReviewFactory.create( page=page, is_active=True, is_complete=True, completed_date=dt3, number_of_violations=30) reviews = domain.get_active_reviews(self.db) expect(reviews).to_length(1) expect(reviews[0].last_review_uuid).to_equal(str(review.uuid))
def test_invalid_review_uuid_returns_redirect(self): page = PageFactory.create() review = ReviewFactory.create(page=page) response = yield self.authenticated_fetch( '/page/%s/review/%s' % (page.uuid, self.ZERO_UUID) ) expect(response.code).to_equal(200) expect(str(review.uuid)).to_equal(loads(response.body).get('uuid'))
def test_can_create_facts_float(self): review = ReviewFactory.create() key = KeyFactory.create(name="some.random.fact") review.add_fact(key, value=1203.01) loaded_review = self.db.query(Review).get(review.id) expect(loaded_review.facts).to_length(1) expect(loaded_review.facts[0].key.name).to_equal("some.random.fact") expect(loaded_review.facts[0].value).to_equal(1203.01)
def test_can_append_violations(self): review = ReviewFactory.build() expect(review.violations).to_length(0) key = KeyFactory.create(name='a') review.add_violation(key, 'b', 100, review.domain) expect(review.violations).to_length(1) expect(review.violations[0].key.name).to_equal('a') expect(review.violations[0].value).to_equal('b') expect(review.violations[0].points).to_equal(100)
def test_can_get_domain_grouped_violations(self): domain1 = DomainFactory.create() domain2 = DomainFactory.create() page1 = PageFactory.create(domain=domain1) page2 = PageFactory.create(domain=domain1) page3 = PageFactory.create(domain=domain2) ReviewFactory.create(domain=domain1, page=page1, is_active=True, number_of_violations=5) ReviewFactory.create(domain=domain1, page=page2, is_active=True, number_of_violations=7) ReviewFactory.create(domain=domain2, page=page3, is_active=True, number_of_violations=9) self.server.application.violation_definitions = { 'key.%s' % i: { 'title': 'title.%s' % i, 'category': 'category.%s' % (i % 3), 'key': Key.get_or_create(self.db, 'key.%d' % i, 'category.%d' % (i % 3)) } for i in xrange(9) } response = yield self.http_client.fetch( self.get_url('/domains/%s/violations' % domain1.name) ) expect(response.code).to_equal(200) domain_violations = loads(response.body) expect(domain_violations).to_length(5) expect(domain_violations.keys()).to_be_like(['domainName', 'violations', 'total', 'domainURL', 'domainId']) expect(domain_violations['total']).to_equal(12) expect(domain_violations['violations']).to_length(3) counts = map(lambda v: v['count'], domain_violations['violations']) expect(counts).to_be_like([5, 4, 3])
def test_can_get_violation_count_and_points(self): domain = DomainFactory.create() domain2 = DomainFactory.create() DomainFactory.create() page = PageFactory.create(domain=domain) page2 = PageFactory.create(domain=domain) page3 = PageFactory.create(domain=domain2) ReviewFactory.create(domain=domain, page=page, is_active=True, number_of_violations=20) ReviewFactory.create(domain=domain, page=page2, is_active=True, number_of_violations=10) ReviewFactory.create(domain=domain2, page=page3, is_active=True, number_of_violations=30) violation_count = domain.get_violation_data(self.db) expect(violation_count).to_equal(30)
def test_can_get_violations_per_day(self): dt = datetime(2013, 10, 10, 10, 10, 10) dt2 = datetime(2013, 10, 11, 10, 10, 10) dt3 = datetime(2013, 10, 12, 10, 10, 10) domain = DomainFactory.create() page = PageFactory.create(domain=domain) ReviewFactory.create(domain=domain, page=page, is_active=False, is_complete=True, completed_date=dt, number_of_violations=20) ReviewFactory.create(domain=domain, page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=10) ReviewFactory.create(domain=domain, page=page, is_active=True, is_complete=True, completed_date=dt3, number_of_violations=30) violations = domain.get_violations_per_day(self.db) expect(violations).to_be_like([ { "completedAt": "2013-10-10", "violation_count": 20, "violation_points": 190 }, { "completedAt": "2013-10-11", "violation_count": 10, "violation_points": 45 }, { "completedAt": "2013-10-12", "violation_count": 30, "violation_points": 435 } ])
def test_can_get_violations_per_day(self): dt = datetime(2013, 10, 10, 10, 10, 10) dt2 = datetime(2013, 10, 11, 10, 10, 10) dt3 = datetime(2013, 10, 12, 10, 10, 10) page = PageFactory.create() ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt, number_of_violations=20) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=10) ReviewFactory.create(page=page, is_active=True, is_complete=True, completed_date=dt3, number_of_violations=30) response = yield self.authenticated_fetch('/domains/%s/violations-per-day/' % page.domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['violations']).to_be_like([ { u'completedAt': u'2013-10-10', u'violation_points': 190, u'violation_count': 20 }, { u'completedAt': u'2013-10-11', u'violation_points': 45, u'violation_count': 10 }, { u'completedAt': u'2013-10-12', u'violation_points': 435, u'violation_count': 30 } ])
def test_can_get_domains_full_data(self): domains = [] for i in xrange(3): domains.append(DomainFactory.create(name='domain-%d.com' % i)) pages = [] for i, domain in enumerate(domains): pages.append([]) for j in xrange(3): pages[i].append(PageFactory.create(domain=domain)) requests = reviews = [] for i, (domain, page) in enumerate(zip(domains, pages)): for j in xrange(i + 1): reviews.append(ReviewFactory.create( domain=domain, page=page[j], is_active=True, number_of_violations=(5 + 2 * j) )) requests.append(RequestFactory.create( status_code=200 if j % 2 == 0 else 404, domain_name=domain.name, response_time=0.25 * (i + 1) )) self.server.application.violation_definitions = { 'key.%s' % i: { 'title': 'title.%s' % i, 'category': 'category.%s' % (i % 3), 'key': Key.get_or_create(self.db, 'key.%d' % i, 'category.%d' % (i % 3)) } for i in xrange(9) } response = yield self.http_client.fetch( self.get_url('/domains-details') ) expect(response.code).to_equal(200) full_data = loads(response.body) expect(full_data).to_length(3) expect(full_data[0].keys()).to_length(10) expect(map(lambda d: d['name'], full_data)).to_be_like(['domain-0.com', 'domain-1.com', 'domain-2.com']) expect(map(lambda d: d['pageCount'], full_data)).to_be_like([3, 3, 3]) expect(map(lambda d: d['reviewCount'], full_data)).to_be_like([1, 2, 3]) expect(map(lambda d: d['violationCount'], full_data)).to_be_like([5, 12, 21]) expect(map(lambda d: d['reviewPercentage'], full_data)).to_be_like([33.33, 66.67, 100.0]) expect(map(lambda d: d['errorPercentage'], full_data)).to_be_like([0.0, 50.0, 33.33]) expect(map(lambda d: d['averageResponseTime'], full_data)).to_be_like([0.25, 0.5, 0.75])
def test_get_violation_count(self): self.sync_cache.redis.delete('g.com-violation-count') gcom = DomainFactory.create(url="http://g.com", name="g.com") page = PageFactory.create(domain=gcom) ReviewFactory.create( is_active=True, is_complete=True, domain=gcom, page=page, number_of_violations=10 ) violation_count = self.sync_cache.get_violation_count(gcom.name) expect(violation_count).to_equal(10) # should get from cache self.sync_cache.db = None violation_count = self.sync_cache.get_violation_count(gcom.name) expect(violation_count).to_equal(10)
def test_can_get_domains_details(self): self.db.query(Domain).delete() details = Domain.get_domains_details(self.db) expect(details).to_length(0) domain = DomainFactory.create(name='domain-1.com', url='http://domain-1.com/') domain2 = DomainFactory.create(name='domain-2.com', url='http://domain-2.com/') DomainFactory.create() page = PageFactory.create(domain=domain) page2 = PageFactory.create(domain=domain) page3 = PageFactory.create(domain=domain2) ReviewFactory.create(domain=domain, page=page, is_active=True, number_of_violations=20) ReviewFactory.create(domain=domain, page=page2, is_active=True, number_of_violations=10) ReviewFactory.create(domain=domain2, page=page3, is_active=True, number_of_violations=30) RequestFactory.create(status_code=200, domain_name=domain.name, response_time=0.25) RequestFactory.create(status_code=304, domain_name=domain.name, response_time=0.35) RequestFactory.create(status_code=400, domain_name=domain.name, response_time=0.25) RequestFactory.create(status_code=403, domain_name=domain.name, response_time=0.35) RequestFactory.create(status_code=404, domain_name=domain.name, response_time=0.25) details = Domain.get_domains_details(self.db) expect(details).to_length(3) expect(details[0]).to_length(10) expect(details[0]['url']).to_equal('http://domain-1.com/') expect(details[0]['name']).to_equal('domain-1.com') expect(details[0]['violationCount']).to_equal(30) expect(details[0]['pageCount']).to_equal(2) expect(details[0]['reviewCount']).to_equal(2) expect(details[0]['reviewPercentage']).to_equal(100.0) expect(details[0]['errorPercentage']).to_equal(60.0) expect(details[0]['is_active']).to_be_true() expect(details[0]['averageResponseTime']).to_equal(0.3)
def test_cant_append_violations_after_complete(self): review = ReviewFactory.build() expect(review.facts).to_length(0) review.is_complete = True try: key = KeyFactory.create(name='a') review.add_violation(key, 'b', 10, review.domain) except ValueError: err = sys.exc_info()[1] expect(err).to_have_an_error_message_of("Can't add anything to a completed review.") else: assert False, 'Should not have gotten this far'
def test_can_get_review(self): dt = datetime(2010, 11, 12, 13, 14, 15) dt_timestamp = calendar.timegm(dt.utctimetuple()) review = ReviewFactory.create(created_date=dt) key1 = KeyFactory.create(name='fact') review.add_fact(key1, 'value') key2 = KeyFactory.create(name='violation') review.add_violation(key2, 'value', 100, review.domain) self.db.flush() response = yield self.authenticated_fetch( '/page/%s/review/%s' % (review.page.uuid, review.uuid)) expect(response.code).to_equal(200) expected = { 'domain': review.domain.name, 'page': review.page.to_dict(), 'uuid': str(review.uuid), 'isComplete': False, 'facts': [{ u'key': u'fact', u'value': u'value', u'title': u'unknown', u'unit': u'value', u'category': u'unknown' }], 'violations': [{ u'points': 100, u'description': u'value', u'key': u'violation', u'title': u'undefined', u'category': 'undefined' }], 'createdAt': dt_timestamp, 'completedAt': None, 'violationPoints': 100, 'violationCount': 1, } expect(loads(response.body)).to_be_like(expected)
def test_can_get_last_reviews_with_domain_filter(self): dt1 = datetime(2010, 10, 10, 10, 10, 10) dt2 = datetime(2010, 10, 11, 10, 10, 10) dt3 = datetime(2010, 10, 12, 10, 10, 10) domain1 = DomainFactory.create() domain2 = DomainFactory.create() page1 = PageFactory.create(domain=domain1) page2 = PageFactory.create(domain=domain2) domain1_review1 = ReviewFactory.create(is_active=True, is_complete=True, page=page1, completed_date=dt1) domain1_review2 = ReviewFactory.create(is_active=True, is_complete=True, page=page1, completed_date=dt2) domain1_review3 = ReviewFactory.create(is_active=True, is_complete=True, page=page1, completed_date=dt3) ReviewFactory.create(is_active=True, is_complete=True, page=page2, completed_date=dt1) ReviewFactory.create(is_active=True, is_complete=True, page=page2, completed_date=dt2) ReviewFactory.create(is_active=True, is_complete=True, page=page2, completed_date=dt3) last_reviews = Review.get_last_reviews(self.db, domain_filter=domain1.name) expect(last_reviews).to_length(3) expect(last_reviews[2].id).to_equal(domain1_review1.id) expect(last_reviews[1].id).to_equal(domain1_review2.id) expect(last_reviews[0].id).to_equal(domain1_review3.id) last_reviews = Review.get_last_reviews(self.db) expect(last_reviews).to_length(6)
def test_can_create_review(self): review = ReviewFactory.create() expect(review.id).not_to_be_null() expect(review.created_date).to_be_like(datetime.utcnow()) expect(review.page).not_to_be_null() expect(review.domain).not_to_be_null() loaded = self.db.query(Review).get(review.id) expect(loaded.created_date).to_be_like(review.created_date) expect(loaded.is_complete).to_be_like(review.is_complete) expect(loaded.uuid).not_to_be_null()
def test_cant_append_violations_after_complete(self): review = ReviewFactory.build() expect(review.facts).to_length(0) review.is_complete = True try: key = KeyFactory.create(name='a') review.add_violation(key, 'b', 10, review.domain) except ValueError: err = sys.exc_info()[1] expect(err).to_have_an_error_message_of( "Can't add anything to a completed review.") else: assert False, 'Should not have gotten this far'
def test_can_process_globo_com(self): self.db.query(Review).delete() self.db.query(Page).delete() self.db.query(Domain).delete() domain = DomainFactory.create(name='globo.com', url='http://www.globo.com') page = PageFactory.create(domain=domain, url='http://www.globo.com/') review = ReviewFactory.create(page=page) self.db.flush() reviewer = self.get_reviewer( api_url='http://localhost:2368', page_uuid=page.uuid, page_url='http://www.globo.com/' ) reviewer._wait_for_async_requests = Mock() reviewer.review() Review.objects.get(review._id, callback=self.stop) loaded_review = self.wait(timeout=30) expect(len(loaded_review.facts)).to_be_greater_than(3) expect(len(loaded_review.violations)).to_be_greater_than(3) print print msg = 'Evaluation for "http://www.globo.com"' print msg print '=' * len(msg) print print 'Facts' print '-----' for fact in loaded_review.facts: print str(fact) print print 'Violations' print '----------' for violation in loaded_review.violations: print str(violation) print print
def test_can_get_domain_reviews_for_next_page(self): dt = datetime(2010, 11, 12, 13, 14, 15) domain = DomainFactory.create(url="http://www.domain-details.com", name="domain-details.com") pages = [] for page_index in range(16): page = PageFactory.create(domain=domain, last_review_date=dt) pages.append(page) reviews = [] for review_index in range(16): review = ReviewFactory.create(page=pages[review_index], is_active=True, is_complete=True, completed_date=dt, number_of_violations=20) reviews.append(review) response = yield self.authenticated_fetch( '/domains/%s/reviews/?current_page=1' % domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['pages']).to_length(10) for i in range(10): expect(domain_details['pages'][i]['url']).to_equal(pages[i].url) expect(domain_details['pages'][i]['uuid']).to_equal( str(pages[i].uuid)) response = yield self.authenticated_fetch( '/domains/%s/reviews/?current_page=2' % domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['pages']).to_length(6) for i in range(6): expect(domain_details['pages'][i]['url']).to_equal(pages[10 + i].url) expect(domain_details['pages'][i]['uuid']).to_equal( str(pages[10 + i].uuid))
def test_to_dict(self): page_id = uuid4() review_id = uuid4() page = PageFactory.build(uuid=page_id) review = ReviewFactory.build(page=page, uuid=review_id) key1 = KeyFactory.create(name='some.random.key1') review.add_violation(key1, 'b', 100, review.domain) key2 = KeyFactory.create(name='some.random.key2') review.add_fact(key2, 'b') fact_definitions = {'some.random.key2': {}} violation_definitions = {'some.random.key1': {}} expect(review.to_dict(fact_definitions, violation_definitions, _)).to_be_like({ 'domain': review.domain.name, 'uuid': str(review_id), 'completedAt': None, 'facts': [{ 'value': 'b', 'key': 'some.random.key2', 'unit': 'value', 'title': 'unknown', 'category': 'unknown' }], 'violations': [{ 'points': 100, 'key': 'some.random.key1', 'description': 'b', 'title': 'undefined', 'category': 'undefined' }], 'page': { 'url': page.url, 'lastModified': None, 'expires': None, 'uuid': str(page_id), 'score': 0.0 }, 'createdAt': review.created_date, 'isComplete': False })
def test_get_by_violation_key_name(self): self.db.query(Review).delete() self.db.query(Violation).delete() review = ReviewFactory.create(is_active=True) for i in range(3): key = Key.get_or_create(self.db, 'violation.%d' % i) review.add_violation(key, 'value', 100, review.domain) review.page.last_review_id = review.id review.page.last_review_uuid = review.uuid review.page.last_review_date = review.completed_date self.db.flush() key_id = review.violations[0].key_id reviews = Review.get_by_violation_key_name(self.db, key_id) expect(reviews).to_length(1)
def test_can_return_reviewer_info(self): review = ReviewFactory.create() reviewer = Reviewer( api_url='http://localhost:2368', page_uuid=review.page.uuid, page_url=review.page.url, page_score=0.0, config=Config(), validators=[] ) validator = Validator(reviewer) expect(validator.page_uuid).to_equal(review.page.uuid) expect(validator.page_url).to_equal(review.page.url) expect(validator.config).to_equal(reviewer.config)
def test_can_get_domain_top_category_violations(self): domain1 = DomainFactory.create() domain2 = DomainFactory.create() page1 = PageFactory.create(domain=domain1) page2 = PageFactory.create(domain=domain1) page3 = PageFactory.create(domain=domain2) ReviewFactory.create(domain=domain1, page=page1, is_active=True, number_of_violations=5) ReviewFactory.create(domain=domain1, page=page2, is_active=True, number_of_violations=7) ReviewFactory.create(domain=domain2, page=page3, is_active=True, number_of_violations=9) self.server.application.violation_definitions = { 'key.%s' % i: { 'title': 'title.%s' % i, 'category': 'category.%s' % (i % 3), 'key': Key.get_or_create(self.db, 'key.%d' % i, 'category.%d' % (i % 3)) } for i in range(9) } key = Key.get_by_name(self.db, 'key.0') response = yield self.authenticated_fetch( '/domains/%s/violations/%d/' % (domain1.name, key.category_id)) expect(response.code).to_equal(200) domain_top_category = loads(response.body) expect(domain_top_category).to_length(5) expect(domain_top_category.keys()).to_be_like([ 'violations', 'domainId', 'categoryId', 'domainURL', 'domainName' ]) expect(domain_top_category['violations']).to_length(3) counts = map(lambda v: v['count'], domain_top_category['violations']) expect(counts).to_be_like([2, 1, 1])
def test_remove_old_reviews(self): self.db.query(Violation).delete() self.db.query(Fact).delete() self.db.query(Key).delete() self.db.query(Review).delete() self.db.query(Page).delete() config = Config() config.NUMBER_OF_REVIEWS_TO_KEEP = 4 dt = datetime(2013, 12, 11, 10, 9, 8) page1 = PageFactory.create() ReviewFactory.create(page=page1, is_active=True, completed_date=dt, number_of_violations=1, number_of_facts=1) page2 = PageFactory.create() ReviewFactory.create(page=page2, is_active=True, completed_date=dt) for x in range(6): dt = datetime(2013, 12, 11, 10, 10, x) ReviewFactory.create(page=page1, is_active=False, completed_date=dt, number_of_violations=2, number_of_facts=1) self.db.flush() reviews = self.db.query(Review).all() expect(reviews).to_length(8) violations = self.db.query(Violation).all() expect(violations).to_length(13) facts = self.db.query(Fact).all() expect(facts).to_length(7) Review.delete_old_reviews(self.db, config, page1) reviews = self.db.query(Review).all() expect(reviews).to_length(6) violations = self.db.query(Violation).all() expect(violations).to_length(9) facts = self.db.query(Fact).all() expect(facts).to_length(5)
def test_can_get_violations_per_day(self): dt = datetime(2013, 10, 10, 10, 10, 10) dt2 = datetime(2013, 10, 11, 10, 10, 10) dt3 = datetime(2013, 10, 12, 10, 10, 10) page = PageFactory.create() ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt, number_of_violations=20) ReviewFactory.create(page=page, is_active=False, is_complete=True, completed_date=dt2, number_of_violations=10) ReviewFactory.create(page=page, is_active=True, is_complete=True, completed_date=dt3, number_of_violations=30) response = yield self.authenticated_fetch( '/domains/%s/violations-per-day/' % page.domain.name) expect(response.code).to_equal(200) domain_details = loads(response.body) expect(domain_details['violations']).to_be_like([{ u'completedAt': u'2013-10-10', u'violation_points': 190, u'violation_count': 20 }, { u'completedAt': u'2013-10-11', u'violation_points': 45, u'violation_count': 10 }, { u'completedAt': u'2013-10-12', u'violation_points': 435, u'violation_count': 30 }])