def process_measure(data, utcnow, session): session_objects = [] measure = Measure() measure.created = utcnow measure.time = data['time'] measure.lat = to_precise_int(data['lat']) measure.lon = to_precise_int(data['lon']) measure.accuracy = data['accuracy'] measure.altitude = data['altitude'] measure.altitude_accuracy = data['altitude_accuracy'] measure.radio = RADIO_TYPE.get(data['radio'], -1) # get measure.id set session.add(measure) session.flush() if data.get('cell'): cells, cell_data = process_cell(data['cell'], measure) measure.cell = dumps(cell_data) session_objects.extend(cells) if data.get('wifi'): # filter out old-style sha1 hashes too_long_keys = False for w in data['wifi']: w['key'] = key = normalize_wifi_key(w['key']) if len(key) > 12: too_long_keys = True break if not too_long_keys: process_wifi(data['wifi'], measure) measure.wifi = dumps(data['wifi']) return (measure, session_objects)
def submit_view(request): api_key = request.GET.get('key', None) heka_client = get_heka_client() if api_key is None: # we don't require API keys for submit yet heka_client.incr('submit.no_api_key') else: heka_client.incr('submit.api_key.%s' % api_key.replace('.', '__')) data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): insert_measures.delay( # TODO convert items to json with support for decimal/datetime items=dumps(items[i:i + 100]), nickname=nickname, ) return HTTPNoContent()
def test_gzip(self): app = self.app data = {"cell": [{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4}]} body = ''.join(gzip_app_iter(dumps(data))) headers = { 'Content-Encoding': 'gzip', } res = app.post('/v1/search?key=test', body, headers=headers, content_type='application/json', status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {"status": "not_found"})
def test_gzip(self): app = self.app data = {"items": [{"lat": 1.0, "lon": 2.0, "wifi": [{"key": "aaaaaaaaaaaa"}]}]} body = ''.join(gzip_app_iter(dumps(data))) headers = { 'Content-Encoding': 'gzip', } res = app.post('/v1/submit?key=test', body, headers=headers, content_type='application/json', status=204) self.assertEqual(res.body, '')
def process_measure(data, utcnow, session, userid=None): measure = Measure() measure.created = utcnow measure.time = data["time"] measure.lat = to_precise_int(data["lat"]) measure.lon = to_precise_int(data["lon"]) measure.accuracy = data["accuracy"] measure.altitude = data["altitude"] measure.altitude_accuracy = data["altitude_accuracy"] measure.radio = RADIO_TYPE.get(data["radio"], -1) # get measure.id set session.add(measure) session.flush() measure_data = dict( id=measure.id, created=encode_datetime(measure.created), lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time), accuracy=measure.accuracy, altitude=measure.altitude, altitude_accuracy=measure.altitude_accuracy, radio=measure.radio, ) if data.get("cell"): insert_cell_measure.delay(measure_data, data["cell"], userid=userid) measure.cell = dumps(data["cell"]) if data.get("wifi"): # filter out old-style sha1 hashes too_long_keys = False for w in data["wifi"]: w["key"] = key = normalize_wifi_key(w["key"]) if len(key) > 12: too_long_keys = True break if not too_long_keys: insert_wifi_measure.delay(measure_data, data["wifi"], userid=userid) measure.wifi = dumps(data["wifi"]) return measure
def submit_view(request): data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): insert_measures.delay( items=dumps(items[i:i + 100]), nickname=nickname, ) return HTTPNoContent()
def submit_view(request): data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): insert_measures.delay( # TODO convert items to json with support for decimal/datetime items=dumps(items[i:i + 100]), nickname=nickname, ) return HTTPNoContent()
def test_gzip(self): app = self.app data = { "items": [{ "lat": 1.0, "lon": 2.0, "wifi": [{ "key": "aaaaaaaaaaaa" }] }] } body = ''.join(gzip_app_iter(dumps(data))) headers = { 'Content-Encoding': 'gzip', } res = app.post('/v1/submit?key=test', body, headers=headers, content_type='application/json', status=204) self.assertEqual(res.body, '')
) from heka.holder import get_client NO_API_KEY = { "error": { "errors": [{ "domain": "usageLimits", "reason": "keyInvalid", "message": "No API key was found", }], "code": 400, "message": "No API key", } } NO_API_KEY = dumps(NO_API_KEY) NOT_FOUND = { "error": { "errors": [{ "domain": "geolocation", "reason": "notFound", "message": "Not found", }], "code": 404, "message": "Not found", } } NOT_FOUND = dumps(NOT_FOUND) PARSE_ERROR = {
ApiKey ) from ichnaea.decimaljson import dumps NO_API_KEY = { "error": { "errors": [{ "domain": "usageLimits", "reason": "keyInvalid", "message": "No API key was found", }], "code": 400, "message": "No API key", } } NO_API_KEY = dumps(NO_API_KEY) def check_api_key(func_name, error_on_invalidkey=False): def c(func): @wraps(func) def closure(request, *args, **kwargs): api_key = request.GET.get('key', None) heka_client = get_heka_client() if api_key is None: heka_client.incr('%s.no_api_key' % func_name) if error_on_invalidkey: result = HTTPBadRequest() result.content_type = 'application/json' result.body = NO_API_KEY
) NO_API_KEY = { "error": { "errors": [{ "domain": "usageLimits", "reason": "keyInvalid", "message": "No API key was found", }], "code": 400, "message": "No API key", } } NO_API_KEY = dumps(NO_API_KEY) NOT_FOUND = { "error": { "errors": [{ "domain": "geolocation", "reason": "notFound", "message": "Not found", }], "code": 404, "message": "Not found", } } NOT_FOUND = dumps(NOT_FOUND)
search_wifi, ) NOT_FOUND = { "error": { "errors": [{ "domain": "geolocation", "reason": "notFound", "message": "Not found", }], "code": 404, "message": "Not found", } } NOT_FOUND = dumps(NOT_FOUND) PARSE_ERROR = { "error": { "errors": [{ "domain": "global", "reason": "parseError", "message": "Parse Error", }], "code": 400, "message": "Parse Error" } } PARSE_ERROR = dumps(PARSE_ERROR)
def __init__(self, errors, status=400): body = {'errors': errors} Response.__init__(self, dumps(body)) self.status = status self.content_type = 'application/json'
logger = logging.getLogger('ichnaea') NOT_FOUND = { "error": { "errors": [{ "domain": "geolocation", "reason": "notFound", "message": "Not found", }], "code": 404, "message": "Not found", } } NOT_FOUND = dumps(NOT_FOUND) PARSE_ERROR = { "error": { "errors": [{ "domain": "global", "reason": "parseError", "message": "Parse Error", }], "code": 400, "message": "Parse Error" } } PARSE_ERROR = dumps(PARSE_ERROR)