def test_can_get_requests_by_status_code(self):
request = RequestFactory.create(
domain_name='globo.com',
status_code=200
)
loaded = Request.get_requests_by_status_code('globo.com', 200, self.db)
expect(loaded[0].url).to_equal(request.url)
expect(loaded[0].review_url).to_equal(request.review_url)
expect(loaded[0].completed_date).to_equal(request.completed_date)
invalid_domain = Request.get_requests_by_status_code(
'g1.globo.com',
200,
self.db
)
expect(invalid_domain).to_equal([])
invalid_code = Request.get_requests_by_status_code(
'globo.com',
2300,
self.db
)
expect(invalid_code).to_equal([])
def test_can_get_requests_by_status_code(self):
request = RequestFactory.create(
domain_name='globo.com',
status_code=200
)
loaded = Request.get_requests_by_status_code('globo.com', 200, self.db)
expect(loaded[0].url).to_equal(request.url)
expect(loaded[0].review_url).to_equal(request.review_url)
expect(loaded[0].completed_date).to_equal(request.completed_date)
invalid_domain = Request.get_requests_by_status_code(
'g1.globo.com',
200,
self.db
)
expect(invalid_domain).to_equal([])
invalid_code = Request.get_requests_by_status_code(
'globo.com',
2300,
self.db
)
expect(invalid_code).to_equal([])
def save_review(cls, page_uuid, review_data, db, search_provider,
fact_definitions, violation_definitions, cache, publish,
config):
from holmes.models import Page, Request
page = Page.by_uuid(page_uuid, db)
if page is None:
return
if review_data['requests']:
Request.save_requests(db, publish, page, review_data['requests'])
last_review = page.last_review
review = Review(
domain_id=page.domain.id,
page_id=page.id,
is_active=True,
is_complete=False,
completed_date=datetime.utcnow(),
uuid=uuid4(),
)
db.add(review)
for fact in review_data['facts']:
name = fact['key']
key = fact_definitions[name]['key']
review.add_fact(key, fact['value'])
for violation in review_data['violations']:
name = violation['key']
key = violation_definitions[name]['key']
review.add_violation(key, violation['value'], violation['points'],
page.domain)
page.expires = review_data['expires']
page.last_modified = review_data['lastModified']
page.last_review_uuid = review.uuid
page.last_review = review
page.last_review_date = review.completed_date
page.violations_count = len(review_data['violations'])
review.is_complete = True
if not last_review:
cache.increment_active_review_count(page.domain)
else:
for violation in last_review.violations:
violation.review_is_active = False
last_review.is_active = False
Review.delete_old_reviews(db, config, page)
search_provider.index_review(review)
publish(dumps({'type': 'new-review', 'reviewId': str(review.uuid)}))
def test_can_get_status_code_info(self):
request = RequestFactory.create(domain_name='g1.globo.com')
loaded = Request.get_status_code_info('g1.globo.com', self.db)
expect(loaded[0].get('code')).to_equal(request.status_code)
expect(loaded[0].get('total')).to_equal(1)
invalid_domain = Request.get_status_code_info('g2.globo.com', self.db)
expect(invalid_domain).to_equal([])
def test_can_get_status_code_info(self):
request = RequestFactory.create(domain_name='g1.globo.com')
loaded = Request.get_status_code_info('g1.globo.com', self.db)
expect(loaded[0].get('code')).to_equal(request.status_code)
expect(loaded[0].get('total')).to_equal(1)
invalid_domain = Request.get_status_code_info('g2.globo.com', self.db)
expect(invalid_domain).to_equal([])
def increment_requests_count(self, increment=1, callback=None):
self.increment_data(
'requests-count',
lambda: Request.get_requests_count(self.db),
increment,
callback
)
def get_requests_count(self, callback=None):
self.get_data(
'requests-count',
int(self.config.REQUESTS_COUNT_EXPIRATION_IN_SECONDS),
lambda: Request.get_requests_count(self.db),
callback=callback
)
def test_can_get_all_status_code(self):
self.db.query(Request).delete()
for i in range(4):
RequestFactory.create(
url='http://m.com/page-%d' % i,
domain_name='m.com',
status_code=200 + (100*i),
completed_date=date.today() - timedelta(days=i)
)
status_code = Request.get_all_status_code(self.db)
expect(status_code).to_length(4)
expect(status_code).to_be_like([
{
'statusCodeTitle': 'OK',
'statusCode': 200
}, {
'statusCodeTitle': 'Multiple Choices',
'statusCode': 300
}, {
'statusCodeTitle': 'Bad Request',
'statusCode': 400
}, {
'statusCodeTitle': 'Internal Server Error',
'statusCode': 500
}
])
def test_can_get_all_status_code(self):
self.db.query(Request).delete()
for i in range(4):
RequestFactory.create(
url='http://m.com/page-%d' % i,
domain_name='m.com',
status_code=200 + (100*i),
completed_date=date.today() - timedelta(days=i)
)
status_code = Request.get_all_status_code(self.db)
expect(status_code).to_length(4)
expect(status_code).to_be_like([
{
'statusCodeTitle': 'OK',
'statusCode': 200
}, {
'statusCodeTitle': 'Multiple Choices',
'statusCode': 300
}, {
'statusCodeTitle': 'Bad Request',
'statusCode': 400
}, {
'statusCodeTitle': 'Internal Server Error',
'statusCode': 500
}
])
def get(self, domain_name, status_code):
if not domain_name:
self.set_status(404, 'Domain %s not found' % domain_name)
return
if not status_code:
self.set_status(404, 'Status code %s not found' % status_code)
return
if status_code == '200':
self.set_status(403, 'Status code %s is not allowed' % status_code)
return
current_page = int(self.get_argument('current_page', 1))
page_size = int(self.get_argument('page_size', 10))
requests = Request.get_requests_by_status_code(
domain_name,
status_code,
self.db,
current_page=current_page,
page_size=page_size
)
requests_count = Request.get_requests_by_status_count(
domain_name,
status_code,
self.db
)
result = {
'statusCodeTitle': get_status_code_title(status_code),
'requestsCount': requests_count,
'requests': []
}
for request in requests:
result['requests'].append({
'id': request.id,
'url': request.url,
'review_url': request.review_url,
'completed_date': request.completed_date
})
self.write_json(result)
def get(self, domain_name, status_code):
if not domain_name:
self.set_status(404, self._('Domain %s not found') % domain_name)
return
if not status_code:
self.set_status(404,
self._('Status code %s not found') % status_code)
return
if status_code == '200':
self.set_status(
403,
self._('Status code %s is not allowed') % status_code)
return
current_page = int(self.get_argument('current_page', 1))
page_size = int(self.get_argument('page_size', 10))
requests = Request.get_requests_by_status_code(
domain_name,
status_code,
self.db,
current_page=current_page,
page_size=page_size)
requests_count = Request.get_requests_by_status_count(
domain_name, status_code, self.db)
result = {
'statusCodeTitle': get_status_code_title(status_code),
'requestsCount': requests_count,
'requests': []
}
for request in requests:
result['requests'].append({
'id': request.id,
'url': request.url,
'review_url': request.review_url,
'completed_date': request.completed_date
})
self.write_json(result)
def test_can_remove_old_requests(self):
self.db.query(Request).delete()
config = Config()
config.DAYS_TO_KEEP_REQUESTS = 1
for i in range(4):
RequestFactory.create(
url='http://m.com/page-%d' % i,
domain_name='m.com',
status_code=200,
completed_date=date.today() - timedelta(days=i)
)
Request.delete_old_requests(self.db, config)
requests = self.db.query(Request).all()
expect(requests).to_length(1)
def test_can_remove_old_requests(self):
self.db.query(Request).delete()
config = Config()
config.DAYS_TO_KEEP_REQUESTS = 1
for i in range(4):
RequestFactory.create(
url='http://m.com/page-%d' % i,
domain_name='m.com',
status_code=200,
completed_date=date.today() - timedelta(days=i)
)
Request.delete_old_requests(self.db, config)
requests = self.db.query(Request).all()
expect(requests).to_length(1)
def get(self):
requests = Request.get_last_requests(
self.db,
current_page=int(self.get_argument('current_page', 1)),
page_size=int(self.get_argument('page_size', 10)),
domain_filter=self.get_argument('domain_filter', None),
status_code_filter=self.get_argument('status_code_filter', None))
self.write_json(
{'requests': [request.to_dict() for request in requests]})
def get(self):
requests = Request.get_last_requests(
self.db,
current_page=int(self.get_argument('current_page', 1)),
page_size=int(self.get_argument('page_size', 10)),
domain_filter=self.get_argument('domain_filter', None),
status_code_filter=self.get_argument('status_code_filter', None)
)
self.write_json(
{'requests': [request.to_dict() for request in requests]}
)
def get(self):
from_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
requests = Request.get_requests_count_by_status_in_period_of_days(self.db, from_date=from_date)
result = []
for request in requests:
result.append({
'statusCode': request.status_code,
'statusCodeTitle': get_status_code_title(request.status_code), # FIXME: is it code or title??
'count': request.count,
})
self.write_json(result)
def test_can_save_requests(self):
self.db.query(Request).delete()
domain = DomainFactory.create(name='t.com')
page = PageFactory.create(domain=domain, url='http://t.com/a.html')
requests = []
for x in range(3):
url = 'http://t.com/file%d.html' % x
response_mock = Mock(
status_code=100 * x,
text='OK',
request_time=0.1 * x,
effective_url=url
)
requests.append((url, response_mock))
publish = Mock()
Request.save_requests(self.db, publish, page, requests)
loaded_requests = self.db.query(Request).all()
expect(loaded_requests).to_length(3)
for idx, request in enumerate(loaded_requests):
expect(request.url).to_equal('http://t.com/file%d.html' % idx)
expect(request.status_code).to_equal(100 * idx)
expect(request.response_time).to_equal(0.1 * idx)
expect(request.domain_name).to_equal('t.com')
expect(request.review_url).to_equal('http://t.com/a.html')
expect(publish.called).to_be_true()
publish.assert_called_once_with(
dumps({'url': url, 'type': 'new-request'})
)
def test_can_get_requests_by_status_count(self):
for i in range(4):
RequestFactory.create(domain_name='globo.com', status_code=200)
total = Request.get_requests_by_status_count(
'globo.com',
200,
self.db
)
expect(total).to_equal(4)
invalid_domain = Request.get_requests_by_status_code(
'g1.globo.com',
200,
self.db
)
expect(invalid_domain).to_equal([])
invalid_code = Request.get_requests_by_status_code(
'globo.com',
2300,
self.db
)
expect(invalid_code).to_equal([])
def test_can_get_requests_by_status_count(self):
for i in range(4):
RequestFactory.create(domain_name='globo.com', status_code=200)
total = Request.get_requests_by_status_count(
'globo.com',
200,
self.db
)
expect(total).to_equal(4)
invalid_domain = Request.get_requests_by_status_code(
'g1.globo.com',
200,
self.db
)
expect(invalid_domain).to_equal([])
invalid_code = Request.get_requests_by_status_code(
'globo.com',
2300,
self.db
)
expect(invalid_code).to_equal([])
def get(self):
current_page = int(self.get_argument('current_page', 1))
page_size = int(self.get_argument('page_size', 10))
requests = Request.get_last_requests(
self.db,
current_page=current_page,
page_size=page_size
)
requests_count = yield self.cache.get_requests_count()
result = {'requestsCount': requests_count, 'requests': []}
for request in requests:
result['requests'].append(request.to_dict())
self.write_json(result)
def get(self, domain_name):
domain = Domain.get_domain_by_name(domain_name, self.db)
if not domain:
self.set_status(404, 'Domain %s not found' % domain_name)
return
page_count = yield self.cache.get_page_count(domain)
review_count = yield self.cache.get_active_review_count(domain)
violation_count = yield self.cache.get_violation_count(domain)
bad_request_count = yield self.cache.get_bad_request_count(domain)
good_request_count = yield self.cache.get_good_request_count(domain)
total_request_count = good_request_count + bad_request_count
if total_request_count > 0:
error_percentage = round(float(bad_request_count) / total_request_count * 100, 2)
else:
error_percentage = 0
response_time_avg = yield self.cache.get_response_time_avg(domain)
status_code_info = Request.get_status_code_info(domain_name, self.db)
if page_count > 0:
review_percentage = round(float(review_count) / page_count * 100, 2)
else:
review_percentage = 0
domain_json = {
"id": domain.id,
"name": domain.name,
"url": domain.url,
"pageCount": page_count,
"reviewCount": review_count,
"violationCount": violation_count,
"reviewPercentage": review_percentage,
"is_active": domain.is_active,
"statusCodeInfo": status_code_info,
"errorPercentage": error_percentage,
"averageResponseTime": response_time_avg,
}
self.write_json(domain_json)
def test_can_get_requests_count_by_status(self):
utcnow = datetime.utcnow()
DomainFactory.create(name='globo.com')
DomainFactory.create(name='globoesporte.com')
DomainFactory.create(name='domain3.com')
for i in range(3):
RequestFactory.create(
status_code=200,
completed_date=utcnow.date() - timedelta(days=i),
domain_name='globo.com'
)
RequestFactory.create(
status_code=404,
completed_date=utcnow.date() - timedelta(days=i),
domain_name='globo.com'
)
RequestFactory.create(
status_code=404,
completed_date=utcnow.date() - timedelta(days=i),
domain_name='globoesporte.com'
)
RequestFactory.create(
status_code=599,
completed_date=utcnow.date() - timedelta(days=i),
)
self.db.flush()
counts = Request.get_requests_count_by_status(self.db)
expect(counts).to_equal({
'_all': [(200, 3), (404, 6)],
u'globo.com': [(200, 3), (404, 3)],
u'domain3.com': [],
u'globoesporte.com': [(404, 3)]
})
def test_can_get_requests_count_by_status(self):
utcnow = datetime.utcnow()
DomainFactory.create(name='globo.com')
DomainFactory.create(name='globoesporte.com')
DomainFactory.create(name='domain3.com')
for i in range(3):
RequestFactory.create(
status_code=200,
completed_date=utcnow.date() - timedelta(days=i),
domain_name='globo.com'
)
RequestFactory.create(
status_code=404,
completed_date=utcnow.date() - timedelta(days=i),
domain_name='globo.com'
)
RequestFactory.create(
status_code=404,
completed_date=utcnow.date() - timedelta(days=i),
domain_name='globoesporte.com'
)
RequestFactory.create(
status_code=599,
completed_date=utcnow.date() - timedelta(days=i),
)
self.db.flush()
counts = Request.get_requests_count_by_status(self.db)
expect(counts).to_equal({
'_all': [(200, 3), (404, 6)],
u'globo.com': [(200, 3), (404, 3)],
u'domain3.com': [],
u'globoesporte.com': [(404, 3)]
})
def _complete_job(self, lock):
self.working_url = None
self.domain_name = None
self._ping_api()
self._release_lock(lock)
Request.delete_old_requests(self.db, self.config)
def save_review(
cls,
page_uuid,
review_data,
db,
search_provider,
fact_definitions,
violation_definitions,
cache,
publish,
config,
):
from holmes.models import Page, Request
page = Page.by_uuid(page_uuid, db)
if page is None:
return
if review_data["requests"]:
Request.save_requests(db, publish, page, review_data["requests"])
last_review = page.last_review
review = Review(
domain_id=page.domain.id,
page_id=page.id,
is_active=True,
is_complete=False,
completed_date=datetime.utcnow(),
uuid=uuid4(),
)
db.add(review)
for fact in review_data["facts"]:
name = fact["key"]
key = fact_definitions[name]["key"]
review.add_fact(key, fact["value"])
for violation in review_data["violations"]:
name = violation["key"]
key = violation_definitions[name]["key"]
review.add_violation(key, violation["value"], violation["points"], page.domain)
page.expires = review_data["expires"]
page.last_modified = review_data["lastModified"]
page.last_review_uuid = review.uuid
page.last_review = review
page.last_review_date = review.completed_date
page.violations_count = len(review_data["violations"])
review.is_complete = True
if not last_review:
cache.increment_active_review_count(page.domain)
else:
for violation in last_review.violations:
violation.review_is_active = False
last_review.is_active = False
Review.delete_old_reviews(db, config, page)
search_provider.index_review(review)
publish(dumps({"type": "new-review", "reviewId": str(review.uuid)}))
def _complete_job(self, lock):
self.working_url = None
self.domain_name = None
self._ping_api()
self._release_lock(lock)
Request.delete_old_requests(self.db, self.config)
def get(self):
status_code = Request.get_all_status_code(self.db)
self.write_json(status_code)
def increment_requests_count(self, increment=1):
self.increment_data(
'requests-count',
lambda: Request.get_requests_count(self.db),
increment
)
def get(self):
status_code = Request.get_all_status_code(self.db)
self.write_json(status_code)
