def preprocess_request(self): request_content = self.request.body if self.request.headers.get("Content-Encoding") == "gzip": # handle gzip self.request bodies try: request_content = util.decode_gzip(self.request.body) except GZIPDecodeError as exc: raise self.prepare_exception(ParseError({"decode": repr(exc)})) try: content = request_content.decode(self.request.charset) except UnicodeDecodeError as exc: # Use str(), since repr() includes the full source bytes raise self.prepare_exception(ParseError({"decode": str(exc)})) request_data = {} if content: try: request_data = json.loads(content) except ValueError as exc: raise self.prepare_exception(ParseError({"decode": repr(exc)})) validated_data = {} errors = None try: validated_data = self.schema.deserialize(request_data) except colander.Invalid as exc: errors = {"validation": exc.asdict()} if request_content and errors: raise self.prepare_exception(ParseError(errors)) return validated_data
def submit(self, api_key): request_data = self.preprocess_request() if not request_data: # don't allow completely empty request raise self.prepare_exception(ParseError()) if not api_key.store_sample("submit"): # only store some percentage of the requests return valid_key = api_key.valid_key data = [] for report in request_data["items"]: source = "gnss" if report is not None: position = report.get("position") if position is not None: source = position.get("source", "gnss") data.append({ "api_key": valid_key, "report": report, "source": source }) self.queue.enqueue(data) self.emit_upload_metrics(len(data), api_key)
def submit(self, api_key): request_data, errors = self.preprocess_request() if not request_data: # don't allow completely empty request raise self.prepare_exception(ParseError()) if not api_key.store_sample('submit'): # only store some percentage of the requests return valid_key = api_key.valid_key data = [] for report in request_data['items']: source = 'gnss' if report is not None: position = report.get('position') if position is not None: source = position.get('source', 'gnss') data.append({ 'api_key': valid_key, 'report': report, 'source': source, }) self.queue.enqueue(data) self.emit_upload_metrics(len(data), api_key)
def check_response(self, data_queues, response, status, fallback=None, details=None): assert response.content_type == "application/json" assert response.headers["Access-Control-Allow-Origin"] == "*" assert response.headers["Access-Control-Max-Age"] == "2592000" if status == "ok": body = dict(response.json) if fallback: assert body["fallback"] == fallback del body["fallback"] assert body == self.ip_response elif status == "invalid_key": assert response.json == InvalidAPIKey().json_body() elif status == "not_found": assert response.json == LocationNotFound().json_body() elif status == "parse_error": assert response.json == ParseError(details).json_body() elif status == "limit_exceeded": assert response.json == DailyLimitExceeded().json_body() if status != "ok": self.check_queue(data_queues, 0)
def preprocess_request(self): errors = [] request_content = self.request.body if self.request.headers.get('Content-Encoding') == 'gzip': # handle gzip self.request bodies try: request_content = util.decode_gzip(self.request.body) except OSError as exc: errors.append({'name': None, 'description': repr(exc)}) request_data = {} try: request_data = json.loads(request_content, encoding=self.request.charset) except ValueError as exc: errors.append({'name': None, 'description': repr(exc)}) validated_data = {} try: validated_data = self.schema.deserialize(request_data) except colander.Invalid as exc: errors.append({'name': None, 'description': exc.asdict()}) if request_content and errors: raise self.prepare_exception(ParseError()) return (validated_data, errors)
def test_error(self, app, celery, raven): wifi = WifiShardFactory.build() res = app.post_json( '/v1/submit', [{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}], status=400) assert res.json == ParseError.json_body()
def preprocess_request(self): errors = [] request_content = self.request.body if self.request.headers.get("Content-Encoding") == "gzip": # handle gzip self.request bodies try: request_content = util.decode_gzip(self.request.body) except GZIPDecodeError as exc: errors.append({"name": None, "description": repr(exc)}) content = request_content.decode(self.request.charset) request_data = {} try: request_data = json.loads(content) except ValueError as exc: errors.append({"name": None, "description": repr(exc)}) validated_data = {} try: validated_data = self.schema.deserialize(request_data) except colander.Invalid as exc: errors.append({"name": None, "description": exc.asdict()}) if request_content and errors: raise self.prepare_exception(ParseError()) return (validated_data, errors)
def test_error_no_mapping(self, app, raven): res = app.post_json(self.url, [1], status=400) detail = { "": ('"[1]" is not a mapping type: Does not implement dict-like' " functionality.") } assert res.json == ParseError({"validation": detail}).json_body()
def test_error(self): wifi = WifiShardFactory.build() res = self.app.post_json( '/v1/submit', [{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}], status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_raven(['ParseError'])
def test_error(self, app, celery, raven): wifi = WifiShardFactory.build() res = app.post_json( '/v1/submit', [{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}], status=400) assert res.json == ParseError.json_body() raven.check(['ParseError'])
def test_error(self, app, celery, raven): wifi = WifiShardFactory.build() res = app.post_json("/v1/submit", [{ "lat": wifi.lat, "lon": wifi.lon, "cell": [] }], status=400) assert res.json == ParseError.json_body()
def test_error_not_dict(self, app, celery, raven): wifi = WifiShardFactory.build() res = app.post_json( "/v1/submit", [{"lat": wifi.lat, "lon": wifi.lon, "cell": []}], status=400 ) detail = { "": ( "\"[{'lat': 51.5, 'lon': -0.1, 'cell': []}]\" is not a mapping" " type: Does not implement dict-like functionality." ) } assert res.json == ParseError({"validation": detail}).json_body()
def check_response(self, response, status): self.assertEqual(response.content_type, 'application/json') self.assertEqual(response.charset, 'UTF-8') if status == 'ok': self.assertEqual(response.json, self.ip_response) elif status == 'invalid_key': self.assertEqual(response.json, InvalidAPIKey.json_body()) elif status == 'not_found': self.assertEqual(response.json, self.not_found.json_body()) elif status == 'parse_error': self.assertEqual(response.json, ParseError.json_body()) elif status == 'limit_exceeded': self.assertEqual(response.json, DailyLimitExceeded.json_body())
def preprocess(self): try: request_data, errors = self.preprocess_request() if not request_data: # don't allow completely empty submit request raise self.prepare_exception(ParseError()) except ParseError: # capture JSON exceptions for submit calls self.raven_client.captureException() raise return request_data
def check_response(self, response, status): self.assertEqual(response.content_type, "application/json") self.assertEqual(response.charset, "UTF-8") self.assertEqual(response.headers["Access-Control-Allow-Origin"], "*") self.assertEqual(response.headers["Access-Control-Max-Age"], "2592000") if status == "ok": self.assertEqual(response.json, self.ip_response) elif status == "invalid_key": self.assertEqual(response.json, InvalidAPIKey.json_body()) elif status == "not_found": self.assertEqual(response.json, self.not_found.json_body()) elif status == "parse_error": self.assertEqual(response.json, ParseError.json_body()) elif status == "limit_exceeded": self.assertEqual(response.json, DailyLimitExceeded.json_body())
def check_response(self, response, status): self.assertEqual(response.content_type, 'application/json') self.assertEqual(response.charset, 'UTF-8') self.assertEqual(response.headers['Access-Control-Allow-Origin'], '*') self.assertEqual(response.headers['Access-Control-Max-Age'], '2592000') if status == 'ok': self.assertEqual(response.json, self.ip_response) elif status == 'invalid_key': self.assertEqual(response.json, InvalidAPIKey.json_body()) elif status == 'not_found': self.assertEqual(response.json, self.not_found.json_body()) elif status == 'parse_error': self.assertEqual(response.json, ParseError.json_body()) elif status == 'limit_exceeded': self.assertEqual(response.json, DailyLimitExceeded.json_body())
def check_response(self, data_queues, response, status): assert response.content_type == 'application/json' assert response.headers['Access-Control-Allow-Origin'] == '*' assert response.headers['Access-Control-Max-Age'] == '2592000' if status == 'ok': assert response.json == self.ip_response elif status == 'invalid_key': assert response.json == InvalidAPIKey.json_body() elif status == 'not_found': assert response.json == self.not_found.json_body() elif status == 'parse_error': assert response.json == ParseError.json_body() elif status == 'limit_exceeded': assert response.json == DailyLimitExceeded.json_body() if status != 'ok': self.check_queue(data_queues, 0)
def transfer(self, api_key): # may raise HTTP error request_data, errors = self.preprocess_request() if not request_data: # don't allow completely empty request raise self.prepare_exception(ParseError()) valid_key = api_key.valid_key data = [] for item in request_data['items']: # pragma: no cover # TODO data.append({ 'api_key': valid_key, 'item': item, }) self.queue.enqueue(data)
def check_response(self, data_queues, response, status, fallback=None): assert response.content_type == 'application/json' assert response.headers['Access-Control-Allow-Origin'] == '*' assert response.headers['Access-Control-Max-Age'] == '2592000' if status == 'ok': body = dict(response.json) if fallback: assert body['fallback'] == fallback del body['fallback'] assert body == self.ip_response elif status == 'invalid_key': assert response.json == InvalidAPIKey.json_body() elif status == 'not_found': assert response.json == self.not_found.json_body() elif status == 'parse_error': assert response.json == ParseError.json_body() elif status == 'limit_exceeded': assert response.json == DailyLimitExceeded.json_body() if status != 'ok': self.check_queue(data_queues, 0)
def test_error_no_json(self, app, raven): res = app.post(self.url, '\xae', status=400) assert res.json == ParseError.json_body()
def test_error_get(self, app, raven): res = app.get(self.url, status=400) assert res.json == ParseError.json_body()
def test_error_empty_body(self): res = self.app.post(self.url, '', status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_raven([('ParseError', 1)])
def test_parse_error(self): res = self.app.post('%s?key=test' % self.url, '\xae', status=400) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, ParseError.json_body()) self.check_stats(counter=[self.metric + '.api_key.test'])
def test_error_no_mapping(self): res = self.app.post_json(self.url, [1], status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_raven([('ParseError', 1)])
def test_error_no_mapping(self): res = self.app.post_json('/v1/search?key=test', [1], status=400) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, ParseError.json_body())
def test_error_no_json(self): res = self.app.post(self.url, '\xae', status=400) self.assertEqual(res.json, ParseError.json_body())
def test_error_no_json(self, app): res = self._call(app, '\xae', method='post', status=400) assert res.json == ParseError.json_body()
def test_error_get(self): res = self.app.get(self.url, status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_raven([("ParseError", 1)])
def test_error_empty_body(self, app, raven): res = app.post(self.url, "", status=400) assert res.json == ParseError().json_body()
def test_error_empty_json(self, app, raven): res = app.post_json(self.url, {}, status=400) assert res.json == ParseError.json_body()
def test_error_no_mapping(self): res = self.app.post_json('/v1/submit', [1], status=400) self.assertEqual(res.json, ParseError.json_body())
def test_error_completely_empty(self): res = self.app.post_json(self.url, [], status=400) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, ParseError.json_body())
def test_error_no_mapping(self, app, raven): res = app.post_json(self.url, [1], status=400) assert res.json == ParseError.json_body()
def test_error_no_mapping(self, app): res = self._call(app, [1], method='post_json', status=400) assert res.json == ParseError.json_body()
def test_no_json(self): res = self.app.post('/v1/search?key=test', '\xae', status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_stats(counter=['search.api_key.test'])
def test_error_get(self, app): res = self._call(app, method='get', status=400) assert res.json == ParseError.json_body()
def test_error_empty_body(self, app, raven): res = app.post(self.url, '', status=400) assert res.json == ParseError.json_body() raven.check([('ParseError', 1)])
def test_error_no_json(self): res = self.app.post(self.url, "\xae", status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_raven([("ParseError", 1)])
def test_error_no_json(self, app, raven): res = app.post(self.url, "\xae", status=400) detail = "JSONDecodeError('Expecting value: line 1 column 1 (char 0)')" assert res.json == ParseError({"decode": detail}).json_body()
def test_error_empty_json(self, app, raven): res = app.post_json(self.url, {}, status=400) detail = {"items": "Required"} assert res.json == ParseError({"validation": detail}).json_body()
def test_error_get(self): res = self.app.get(self.url, status=400) self.assertEqual(res.json, ParseError.json_body())
def test_error_empty_json(self): res = self.app.post_json(self.url, {}, status=400) self.assertEqual(res.json, ParseError.json_body())