def mock_url_check(url, data=None, status=200):
url_hash = faker.md5()
httpretty.register_uri(httpretty.POST, CHECK_ONE_URL,
body=json.dumps({'url-hash': url_hash}),
content_type='application/json')
check_url = '/'.join((METADATA_URL, url_hash))
httpretty.register_uri(httpretty.GET, check_url,
body=metadata_factory(url, data),
content_type='application/json',
status=status)
def hit_factory():
return {
"_score": float(faker.random_number(2)),
"_type": "fake",
"_id": faker.md5(),
"_source": {
"title": faker.sentence(),
"tags": [faker.word() for _ in range(faker.random_digit())]
},
"_index": "udata-test"
}
def hit_factory():
return {
"_score": float(faker.random_number(2)),
"_type": "fake",
"_id": faker.md5(),
"_source": {
"title": faker.sentence(),
"tags": [faker.word() for _ in range(faker.random_digit())]
},
"_index": "udata-test"
}
def test_json_error_check_url(self, httpretty):
url_hash = faker.md5()
httpretty.register_uri(httpretty.POST,
CHECK_ONE_URL,
body=json.dumps({'url-hash': url_hash}),
content_type='application/json')
check_url = '/'.join((METADATA_URL, url_hash))
httpretty.register_uri(httpretty.GET,
check_url,
body='<strong>not json</strong>',
content_type='text/html')
res = self.checker.check(self.resource)
assert res is None
def test_json_error_check_url(self):
url = faker.uri()
url_hash = faker.md5()
httpretty.register_uri(httpretty.POST, CHECK_ONE_URL,
body=json.dumps({'url-hash': url_hash}),
content_type='application/json')
check_url = '/'.join((METADATA_URL, url_hash))
httpretty.register_uri(httpretty.GET, check_url,
body='<strong>not json</strong>',
content_type='test/html')
response = self.get(url_for('api.checkurl'),
qs={'url': url, 'group': ''})
self.assertStatus(response, 503)
self.assertIn('error', response.json)
def metadata_factory(url, data=None):
response = {
'etag': '',
'url': url,
'content-length': faker.pyint(),
'content-disposition': '',
'content-md5': faker.md5(),
'content-location': '',
'expires': faker.iso8601(),
'status': 200,
'updated': faker.iso8601(),
'last-modified': faker.iso8601(),
'content-encoding': 'gzip',
'content-type': faker.mime_type()
}
if data:
response.update(data)
return json.dumps(response)
def metadata_factory(url, data=None):
"""Base for a mocked Croquemort HTTP response"""
response = {
'etag': '',
'checked-url': url,
'content-length': faker.pyint(),
'content-disposition': '',
'content-md5': faker.md5(),
'content-location': '',
'expires': faker.iso8601(),
'final-status-code': 200,
'updated': faker.iso8601(),
'last-modified': faker.iso8601(),
'content-encoding': 'gzip',
'content-type': faker.mime_type()
}
if data:
response.update(data)
return json.dumps(response)
def test_post_request(self, httpretty):
url = self.resource.url
url_hash = faker.md5()
httpretty.register_uri(httpretty.POST,
CHECK_ONE_URL,
body=json.dumps({'url-hash': url_hash}),
content_type='application/json')
check_url = '/'.join((METADATA_URL, url_hash))
httpretty.register_uri(httpretty.GET,
check_url,
body=metadata_factory(url),
content_type='application/json',
status=200)
self.checker.check(self.resource)
assert len(httpretty.core.httpretty.latest_requests)
post_request = httpretty.core.httpretty.latest_requests[0]
assert json.loads(post_request.body) == {
'url': self.resource.url,
'group': self.dataset.slug
}
def test_retry(self, httpretty):
'''Test the `is_pending` logic from utils.check_url'''
url = self.resource.url
url_hash = faker.md5()
httpretty.register_uri(httpretty.POST,
CHECK_ONE_URL,
body=json.dumps({'url-hash': url_hash}),
content_type='application/json')
check_url = '/'.join((METADATA_URL, url_hash))
def make_response(status, body=None):
return httpretty.Response(body=body or metadata_factory(url),
content_type='application/json',
status=status)
httpretty.register_uri(httpretty.GET,
check_url,
responses=[
make_response(500),
make_response(404, body=''),
make_response(200)
])
res = self.checker.check(self.resource)
assert res['check:status'], 200