def preprocess_request(self):
request_content = self.request.body
if self.request.headers.get("Content-Encoding") == "gzip":
# handle gzip self.request bodies
try:
request_content = util.decode_gzip(self.request.body)
except GZIPDecodeError as exc:
raise self.prepare_exception(ParseError({"decode": repr(exc)}))
try:
content = request_content.decode(self.request.charset)
except UnicodeDecodeError as exc:
# Use str(), since repr() includes the full source bytes
raise self.prepare_exception(ParseError({"decode": str(exc)}))
request_data = {}
if content:
try:
request_data = json.loads(content)
except ValueError as exc:
raise self.prepare_exception(ParseError({"decode": repr(exc)}))
validated_data = {}
errors = None
try:
validated_data = self.schema.deserialize(request_data)
except colander.Invalid as exc:
errors = {"validation": exc.asdict()}
if request_content and errors:
raise self.prepare_exception(ParseError(errors))
return validated_data
def submit(self, api_key):
request_data = self.preprocess_request()
if not request_data:
# don't allow completely empty request
raise self.prepare_exception(ParseError())
if not api_key.store_sample("submit"):
# only store some percentage of the requests
return
valid_key = api_key.valid_key
data = []
for report in request_data["items"]:
source = "gnss"
if report is not None:
position = report.get("position")
if position is not None:
source = position.get("source", "gnss")
data.append({
"api_key": valid_key,
"report": report,
"source": source
})
self.queue.enqueue(data)
self.emit_upload_metrics(len(data), api_key)
def submit(self, api_key):
request_data, errors = self.preprocess_request()
if not request_data:
# don't allow completely empty request
raise self.prepare_exception(ParseError())
if not api_key.store_sample('submit'):
# only store some percentage of the requests
return
valid_key = api_key.valid_key
data = []
for report in request_data['items']:
source = 'gnss'
if report is not None:
position = report.get('position')
if position is not None:
source = position.get('source', 'gnss')
data.append({
'api_key': valid_key,
'report': report,
'source': source,
})
self.queue.enqueue(data)
self.emit_upload_metrics(len(data), api_key)
def check_response(self,
data_queues,
response,
status,
fallback=None,
details=None):
assert response.content_type == "application/json"
assert response.headers["Access-Control-Allow-Origin"] == "*"
assert response.headers["Access-Control-Max-Age"] == "2592000"
if status == "ok":
body = dict(response.json)
if fallback:
assert body["fallback"] == fallback
del body["fallback"]
assert body == self.ip_response
elif status == "invalid_key":
assert response.json == InvalidAPIKey().json_body()
elif status == "not_found":
assert response.json == LocationNotFound().json_body()
elif status == "parse_error":
assert response.json == ParseError(details).json_body()
elif status == "limit_exceeded":
assert response.json == DailyLimitExceeded().json_body()
if status != "ok":
self.check_queue(data_queues, 0)
def preprocess_request(self):
errors = []
request_content = self.request.body
if self.request.headers.get('Content-Encoding') == 'gzip':
# handle gzip self.request bodies
try:
request_content = util.decode_gzip(self.request.body)
except OSError as exc:
errors.append({'name': None, 'description': repr(exc)})
request_data = {}
try:
request_data = json.loads(request_content,
encoding=self.request.charset)
except ValueError as exc:
errors.append({'name': None, 'description': repr(exc)})
validated_data = {}
try:
validated_data = self.schema.deserialize(request_data)
except colander.Invalid as exc:
errors.append({'name': None, 'description': exc.asdict()})
if request_content and errors:
raise self.prepare_exception(ParseError())
return (validated_data, errors)
def test_error(self, app, celery, raven):
wifi = WifiShardFactory.build()
res = app.post_json(
'/v1/submit',
[{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}],
status=400)
assert res.json == ParseError.json_body()
def preprocess_request(self):
errors = []
request_content = self.request.body
if self.request.headers.get("Content-Encoding") == "gzip":
# handle gzip self.request bodies
try:
request_content = util.decode_gzip(self.request.body)
except GZIPDecodeError as exc:
errors.append({"name": None, "description": repr(exc)})
content = request_content.decode(self.request.charset)
request_data = {}
try:
request_data = json.loads(content)
except ValueError as exc:
errors.append({"name": None, "description": repr(exc)})
validated_data = {}
try:
validated_data = self.schema.deserialize(request_data)
except colander.Invalid as exc:
errors.append({"name": None, "description": exc.asdict()})
if request_content and errors:
raise self.prepare_exception(ParseError())
return (validated_data, errors)
def test_error_no_mapping(self, app, raven):
res = app.post_json(self.url, [1], status=400)
detail = {
"": ('"[1]" is not a mapping type: Does not implement dict-like'
" functionality.")
}
assert res.json == ParseError({"validation": detail}).json_body()
def test_error(self):
wifi = WifiShardFactory.build()
res = self.app.post_json(
'/v1/submit',
[{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}],
status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven(['ParseError'])
def test_error(self, app, celery, raven):
wifi = WifiShardFactory.build()
res = app.post_json(
'/v1/submit',
[{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}],
status=400)
assert res.json == ParseError.json_body()
raven.check(['ParseError'])
def test_error(self):
wifi = WifiShardFactory.build()
res = self.app.post_json(
'/v1/submit',
[{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}],
status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven(['ParseError'])
def test_error(self, app, celery, raven):
wifi = WifiShardFactory.build()
res = app.post_json("/v1/submit", [{
"lat": wifi.lat,
"lon": wifi.lon,
"cell": []
}],
status=400)
assert res.json == ParseError.json_body()
def test_error_not_dict(self, app, celery, raven):
wifi = WifiShardFactory.build()
res = app.post_json(
"/v1/submit", [{"lat": wifi.lat, "lon": wifi.lon, "cell": []}], status=400
)
detail = {
"": (
"\"[{'lat': 51.5, 'lon': -0.1, 'cell': []}]\" is not a mapping"
" type: Does not implement dict-like functionality."
)
}
assert res.json == ParseError({"validation": detail}).json_body()
def check_response(self, response, status):
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(response.charset, 'UTF-8')
if status == 'ok':
self.assertEqual(response.json, self.ip_response)
elif status == 'invalid_key':
self.assertEqual(response.json, InvalidAPIKey.json_body())
elif status == 'not_found':
self.assertEqual(response.json, self.not_found.json_body())
elif status == 'parse_error':
self.assertEqual(response.json, ParseError.json_body())
elif status == 'limit_exceeded':
self.assertEqual(response.json, DailyLimitExceeded.json_body())
def preprocess(self):
try:
request_data, errors = self.preprocess_request()
if not request_data:
# don't allow completely empty submit request
raise self.prepare_exception(ParseError())
except ParseError:
# capture JSON exceptions for submit calls
self.raven_client.captureException()
raise
return request_data
def check_response(self, response, status):
self.assertEqual(response.content_type, "application/json")
self.assertEqual(response.charset, "UTF-8")
self.assertEqual(response.headers["Access-Control-Allow-Origin"], "*")
self.assertEqual(response.headers["Access-Control-Max-Age"], "2592000")
if status == "ok":
self.assertEqual(response.json, self.ip_response)
elif status == "invalid_key":
self.assertEqual(response.json, InvalidAPIKey.json_body())
elif status == "not_found":
self.assertEqual(response.json, self.not_found.json_body())
elif status == "parse_error":
self.assertEqual(response.json, ParseError.json_body())
elif status == "limit_exceeded":
self.assertEqual(response.json, DailyLimitExceeded.json_body())
def check_response(self, response, status):
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(response.charset, 'UTF-8')
self.assertEqual(response.headers['Access-Control-Allow-Origin'], '*')
self.assertEqual(response.headers['Access-Control-Max-Age'], '2592000')
if status == 'ok':
self.assertEqual(response.json, self.ip_response)
elif status == 'invalid_key':
self.assertEqual(response.json, InvalidAPIKey.json_body())
elif status == 'not_found':
self.assertEqual(response.json, self.not_found.json_body())
elif status == 'parse_error':
self.assertEqual(response.json, ParseError.json_body())
elif status == 'limit_exceeded':
self.assertEqual(response.json, DailyLimitExceeded.json_body())
def check_response(self, response, status):
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(response.charset, 'UTF-8')
self.assertEqual(response.headers['Access-Control-Allow-Origin'], '*')
self.assertEqual(response.headers['Access-Control-Max-Age'], '2592000')
if status == 'ok':
self.assertEqual(response.json, self.ip_response)
elif status == 'invalid_key':
self.assertEqual(response.json, InvalidAPIKey.json_body())
elif status == 'not_found':
self.assertEqual(response.json, self.not_found.json_body())
elif status == 'parse_error':
self.assertEqual(response.json, ParseError.json_body())
elif status == 'limit_exceeded':
self.assertEqual(response.json, DailyLimitExceeded.json_body())
def check_response(self, data_queues, response, status):
assert response.content_type == 'application/json'
assert response.headers['Access-Control-Allow-Origin'] == '*'
assert response.headers['Access-Control-Max-Age'] == '2592000'
if status == 'ok':
assert response.json == self.ip_response
elif status == 'invalid_key':
assert response.json == InvalidAPIKey.json_body()
elif status == 'not_found':
assert response.json == self.not_found.json_body()
elif status == 'parse_error':
assert response.json == ParseError.json_body()
elif status == 'limit_exceeded':
assert response.json == DailyLimitExceeded.json_body()
if status != 'ok':
self.check_queue(data_queues, 0)
def transfer(self, api_key):
# may raise HTTP error
request_data, errors = self.preprocess_request()
if not request_data:
# don't allow completely empty request
raise self.prepare_exception(ParseError())
valid_key = api_key.valid_key
data = []
for item in request_data['items']: # pragma: no cover
# TODO
data.append({
'api_key': valid_key,
'item': item,
})
self.queue.enqueue(data)
def check_response(self, data_queues, response, status, fallback=None):
assert response.content_type == 'application/json'
assert response.headers['Access-Control-Allow-Origin'] == '*'
assert response.headers['Access-Control-Max-Age'] == '2592000'
if status == 'ok':
body = dict(response.json)
if fallback:
assert body['fallback'] == fallback
del body['fallback']
assert body == self.ip_response
elif status == 'invalid_key':
assert response.json == InvalidAPIKey.json_body()
elif status == 'not_found':
assert response.json == self.not_found.json_body()
elif status == 'parse_error':
assert response.json == ParseError.json_body()
elif status == 'limit_exceeded':
assert response.json == DailyLimitExceeded.json_body()
if status != 'ok':
self.check_queue(data_queues, 0)
def test_error_no_json(self, app, raven):
res = app.post(self.url, '\xae', status=400)
assert res.json == ParseError.json_body()
def test_error_get(self, app, raven):
res = app.get(self.url, status=400)
assert res.json == ParseError.json_body()
def test_error_empty_body(self):
res = self.app.post(self.url, '', status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven([('ParseError', 1)])
def test_parse_error(self):
res = self.app.post('%s?key=test' % self.url, '\xae', status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, ParseError.json_body())
self.check_stats(counter=[self.metric + '.api_key.test'])
def test_error_no_mapping(self):
res = self.app.post_json(self.url, [1], status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven([('ParseError', 1)])
def test_error_no_mapping(self):
res = self.app.post_json('/v1/search?key=test', [1], status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, ParseError.json_body())
def test_error_no_json(self):
res = self.app.post(self.url, '\xae', status=400)
self.assertEqual(res.json, ParseError.json_body())
def test_error_no_json(self, app):
res = self._call(app, '\xae', method='post', status=400)
assert res.json == ParseError.json_body()
def test_error_get(self):
res = self.app.get(self.url, status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven([("ParseError", 1)])
def test_error_empty_body(self, app, raven):
res = app.post(self.url, "", status=400)
assert res.json == ParseError().json_body()
def test_error_get(self, app, raven):
res = app.get(self.url, status=400)
assert res.json == ParseError.json_body()
def test_error_empty_json(self, app, raven):
res = app.post_json(self.url, {}, status=400)
assert res.json == ParseError.json_body()
def test_error_no_mapping(self):
res = self.app.post_json('/v1/submit', [1], status=400)
self.assertEqual(res.json, ParseError.json_body())
def test_error_completely_empty(self):
res = self.app.post_json(self.url, [], status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, ParseError.json_body())
def test_error_no_mapping(self, app, raven):
res = app.post_json(self.url, [1], status=400)
assert res.json == ParseError.json_body()
def test_error_no_mapping(self, app):
res = self._call(app, [1], method='post_json', status=400)
assert res.json == ParseError.json_body()
def test_no_json(self):
res = self.app.post('/v1/search?key=test', '\xae', status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_stats(counter=['search.api_key.test'])
def test_error_get(self, app):
res = self._call(app, method='get', status=400)
assert res.json == ParseError.json_body()
def test_error_empty_body(self):
res = self.app.post(self.url, '', status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven([('ParseError', 1)])
def test_error_no_json(self, app):
res = self._call(app, '\xae', method='post', status=400)
assert res.json == ParseError.json_body()
def test_error_empty_body(self, app, raven):
res = app.post(self.url, '', status=400)
assert res.json == ParseError.json_body()
raven.check([('ParseError', 1)])
def test_error_no_mapping(self, app):
res = self._call(app, [1], method='post_json', status=400)
assert res.json == ParseError.json_body()
def test_error_no_json(self):
res = self.app.post(self.url, "\xae", status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven([("ParseError", 1)])
def test_error_no_json(self, app, raven):
res = app.post(self.url, "\xae", status=400)
detail = "JSONDecodeError('Expecting value: line 1 column 1 (char 0)')"
assert res.json == ParseError({"decode": detail}).json_body()
def test_error_no_mapping(self):
res = self.app.post_json(self.url, [1], status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven([('ParseError', 1)])
def test_error_empty_json(self, app, raven):
res = app.post_json(self.url, {}, status=400)
detail = {"items": "Required"}
assert res.json == ParseError({"validation": detail}).json_body()
def test_error_empty_json(self, app, raven):
res = app.post_json(self.url, {}, status=400)
assert res.json == ParseError.json_body()
def test_error_get(self):
res = self.app.get(self.url, status=400)
self.assertEqual(res.json, ParseError.json_body())
def test_error_no_mapping(self, app, raven):
res = app.post_json(self.url, [1], status=400)
assert res.json == ParseError.json_body()
def test_error_empty_json(self):
res = self.app.post_json(self.url, {}, status=400)
self.assertEqual(res.json, ParseError.json_body())
def test_error_get(self, app):
res = self._call(app, method='get', status=400)
assert res.json == ParseError.json_body()
def test_error_no_json(self, app, raven):
res = app.post(self.url, '\xae', status=400)
assert res.json == ParseError.json_body()
