def process_single(request): stats_client = get_stats_client() locate_data, locate_errors = preprocess_request( request, schema=GeoLocateSchema(), extra_checks=(geolocate_validator, ), response=JSONParseError, accept_empty=True, ) data, errors = preprocess_request( request, schema=GeoSubmitSchema(), extra_checks=(geosubmit_validator, ), response=None, ) data = {'items': [data]} nickname = request.headers.get('X-Nickname', u'') email = request.headers.get('X-Email', u'') upload_items = flatten_items(data) errors = process_upload(nickname, email, upload_items) if errors is not SENTINEL and errors: # pragma: no cover stats_client.incr('geosubmit.upload.errors', len(errors)) first_item = data['items'][0] if first_item['latitude'] == -255 or first_item['longitude'] == -255: data = map_data(data['items'][0]) session = request.db_slave_session result = search_all_sources( session, 'geosubmit', data, client_addr=request.client_addr, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None)) else: result = { 'lat': first_item['latitude'], 'lon': first_item['longitude'], 'accuracy': first_item['accuracy'] } if result is None: stats_client.incr('geosubmit.miss') result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { "location": { "lat": result['lat'], "lng": result['lon'], }, "accuracy": float(result['accuracy']), }
def process_single(request): stats_client = get_stats_client() locate_data, locate_errors = preprocess_request( request, schema=GeoLocateSchema(), extra_checks=(geolocate_validator, ), response=JSONParseError, accept_empty=True, ) data, errors = preprocess_request( request, schema=GeoSubmitSchema(), extra_checks=(geosubmit_validator,), response=None, ) data = {'items': [data]} nickname = request.headers.get('X-Nickname', u'') email = request.headers.get('X-Email', u'') upload_items = flatten_items(data) errors = process_upload(nickname, email, upload_items) if errors is not SENTINEL and errors: # pragma: no cover stats_client.incr('geosubmit.upload.errors', len(errors)) first_item = data['items'][0] if first_item['latitude'] == -255 or first_item['longitude'] == -255: data = map_data(data['items'][0]) session = request.db_slave_session result = search_all_sources( session, 'geosubmit', data, client_addr=request.client_addr, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None)) else: result = {'lat': first_item['latitude'], 'lon': first_item['longitude'], 'accuracy': first_item['accuracy']} if result is None: stats_client.incr('geosubmit.miss') result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { "location": { "lat": result['lat'], "lng": result['lon'], }, "accuracy": float(result['accuracy']), }
def geolocate_view(request): request_data, errors = preprocess_request( request, schema=GeoLocateSchema(), response=JSONParseError, accept_empty=True, ) search_data = prepare_search_data( request_data, client_addr=request.client_addr) result = PositionSearcher( session_db=request.db_ro_session, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None), api_name='geolocate', ).search(search_data) if not result: result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { 'location': { 'lat': result['lat'], 'lng': result['lon'], }, 'accuracy': float(result['accuracy']), }
def submit_view(request): api_key = request.GET.get('key', None) heka_client = get_heka_client() if api_key is None: # we don't require API keys for submit yet heka_client.incr('submit.no_api_key') else: heka_client.incr('submit.api_key.%s' % api_key.replace('.', '__')) data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): insert_measures.delay( # TODO convert items to json with support for decimal/datetime items=dumps(items[i:i + 100]), nickname=nickname, ) return HTTPNoContent()
def submit_view(request): data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): items = dumps(items[i:i + 100]) # insert measures, expire the task if it wasn't processed # after two hours to avoid queue overload try: insert_measures.apply_async( kwargs={'items': items, 'nickname': nickname}, expires=7200) except ConnectionError: return HTTPServiceUnavailable() return HTTPNoContent()
def country_view(request): request_data, errors = preprocess_request( request, schema=GeoLocateSchema(), response=JSONParseError, accept_empty=True, ) search_data = prepare_search_data( request_data, client_addr=request.client_addr) response = CountrySearcher( session_db=request.db_ro_session, geoip_db=request.registry.geoip_db, api_key_log=False, api_key_name=None, api_name='country', ).search(search_data) if not response: response = HTTPNotFound() response.content_type = 'application/json' response.body = NOT_FOUND return response return response
def geolocate_view(request): data, errors = preprocess_request( request, schema=GeoLocateSchema(), response=JSONParseError, accept_empty=True, ) data = map_data(data) session = request.db_slave_session result = search_all_sources(session, 'geolocate', data, client_addr=request.client_addr, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None)) if not result: result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { "location": { "lat": result['lat'], "lng": result['lon'], }, "accuracy": float(result['accuracy']), }
def search_view(request): data, errors = preprocess_request( request, schema=SearchSchema(), extra_checks=(check_cell_or_wifi, ), accept_empty=True, ) session = request.db_slave_session result = search_all_sources( session, 'search', data, client_addr=request.client_addr, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None)) if not result: return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def test_empty(self): schema = self._make_schema() request = self._make_request('{}') data, errors = preprocess_request(request, schema, response=None) self.assertEqual(errors, []) self.assertEqual(data, {'cell': (), 'wifi': (), 'radio': None})
def search_view(request, api_key): request_data, errors = preprocess_request( request, schema=SearchSchema(), accept_empty=True, ) search_data = prepare_search_data( request_data, client_addr=request.client_addr) result = PositionSearcher( session_db=request.db_ro_session, geoip_db=request.registry.geoip_db, api_key=api_key, api_name='search', ).search(search_data) if not result: return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def search_view(request): data, errors = preprocess_request( request, schema=SearchSchema(), accept_empty=True, ) data['geoip'] = request.client_addr session = request.db_slave_session result = PositionSearcher( { 'geoip': request.registry.geoip_db, 'session': session }, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None), api_name='search', ).search(data) if not result: return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def geolocate_view(request, api_key): request_data, errors = preprocess_request( request, schema=GeoLocateSchema(), response=JSONParseError, accept_empty=True, ) search_data = prepare_search_data(request_data, client_addr=request.client_addr) result = PositionSearcher( session_db=request.db_ro_session, geoip_db=request.registry.geoip_db, api_key=api_key, api_name='geolocate', ).search(search_data) if not result: result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { 'location': { 'lat': result['lat'], 'lng': result['lon'], }, 'accuracy': float(result['accuracy']), }
def geolocate_view(request): data, errors = preprocess_request( request, schema=GeoLocateSchema(), extra_checks=(geolocate_validator, ), response=JSONParseError, accept_empty=True, ) data = map_data(data) session = request.db_slave_session result = search_all_sources( session, 'geolocate', data, client_addr=request.client_addr, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None)) if not result: result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { "location": { "lat": result['lat'], "lng": result['lon'], }, "accuracy": float(result['accuracy']), }
def test_minimal(self): schema = self._make_schema() request = self._make_request( '{"items": [{"lat": 12.3456781, "lon": 23.4567892}]}') data, errors = preprocess_request(request, schema, response=None) self.assertFalse(errors) self.assertTrue('items' in data) self.assertEqual(len(data['items']), 1)
def geosubmit_view(request): stats_client = request.registry.stats_client api_key_log = getattr(request, 'api_key_log', False) api_key_name = getattr(request, 'api_key_name', None) try: data, errors = preprocess_request( request, schema=GeoSubmitBatchSchema(), response=JSONParseError, ) except JSONParseError: # capture JSON exceptions for submit calls request.registry.heka_client.raven(RAVEN_ERROR) raise items = map_items(data['items']) nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') email = request.headers.get('X-Email', u'') if isinstance(email, str): email = email.decode('utf-8', 'ignore') # count the number of batches and emit a pseudo-timer to capture # the number of reports per batch length = len(items) stats_client.incr('items.uploaded.batches') stats_client.timing('items.uploaded.batch_size', length) if api_key_log: stats_client.incr('items.api_log.%s.uploaded.batches' % api_key_name) stats_client.timing( 'items.api_log.%s.uploaded.batch_size' % api_key_name, length) # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, length, 100): batch = kombu_dumps(items[i:i + 100]) # insert observations, expire the task if it wasn't processed # after six hours to avoid queue overload try: insert_measures.apply_async(kwargs={ 'email': email, 'items': batch, 'nickname': nickname, 'api_key_log': api_key_log, 'api_key_name': api_key_name, }, expires=21600) except ConnectionError: # pragma: no cover return HTTPServiceUnavailable() result = HTTPOk() result.content_type = 'application/json' result.body = '{}' return result
def geolocate_view(request): heka_client = get_heka_client() data, errors = preprocess_request( request, schema=GeoLocateSchema(), extra_checks=(geolocate_validator, ), response=JSONError, accept_empty=True, ) session = request.db_slave_session result = None if data and data['wifiAccessPoints']: result = search_wifi_ap(session, data) if result is not None: heka_client.incr('geolocate.wifi_hit') heka_client.timer_send('geolocate.accuracy.wifi', result['accuracy']) elif data: result = search_cell_tower(session, data) if result is not None: heka_client.incr('geolocate.cell_hit') heka_client.timer_send('geolocate.accuracy.cell', result['accuracy']) if result is None: result = search_cell_tower_lac(session, data) if result is not None: heka_client.incr('geolocate.cell_lac_hit') heka_client.timer_send('geolocate.accuracy.cell_lac', result['accuracy']) if result is None and request.client_addr: result = search_geoip(request.registry.geoip_db, request.client_addr) if result is not None: heka_client.incr('geolocate.geoip_hit') heka_client.timer_send('geolocate.accuracy.geoip', result['accuracy']) if result is None: heka_client.incr('geolocate.miss') result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { "location": { "lat": result['lat'], "lng": result['lon'], }, "accuracy": float(result['accuracy']), }
def search_view(request): heka_client = get_heka_client() data, errors = preprocess_request( request, schema=SearchSchema(), extra_checks=(check_cell_or_wifi, ), accept_empty=True, ) session = request.db_slave_session result = None if data and data['wifi']: result = search_wifi(session, data) if result is not None: heka_client.incr('search.wifi_hit') heka_client.timer_send('search.accuracy.wifi', result['accuracy']) if result is None and data: # no wifi result found, fall back to cell result = search_cell(session, data) if result is not None: heka_client.incr('search.cell_hit') heka_client.timer_send('search.accuracy.cell', result['accuracy']) if result is None and data: # no direct cell result found, try cell LAC result = search_cell_lac(session, data) if result is not None: heka_client.incr('search.cell_lac_hit') heka_client.timer_send('search.accuracy.cell_lac', result['accuracy']) if result is None and request.client_addr: # no cell or wifi, fall back again to geoip result = search_geoip(request.registry.geoip_db, request.client_addr) if result is not None: heka_client.incr('search.geoip_hit') heka_client.timer_send('search.accuracy.geoip', result['accuracy']) if result is None: heka_client.incr('search.miss') return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def geolocate_view(request): api_key = request.GET.get('key', None) heka_client = get_heka_client() if api_key is None: heka_client.incr('geolocate.no_api_key') result = HTTPBadRequest() result.content_type = 'application/json' result.body = NO_API_KEY return result heka_client.incr('geolocate.api_key.%s' % api_key.replace('.', '__')) data, errors = preprocess_request( request, schema=GeoLocateSchema(), extra_checks=(geolocate_validator, ), response=JSONError, ) session = request.db_slave_session result = None if data['wifiAccessPoints']: result = search_wifi_ap(session, data) if result is not None: heka_client.incr('geolocate.wifi_hit') else: result = search_cell_tower(session, data) if result is not None: heka_client.incr('geolocate.cell_hit') if result is None and request.client_addr: result = search_geoip(request.registry.geoip_db, request.client_addr) if result is not None: heka_client.incr('geolocate.geoip_hit') if result is None: heka_client.incr('geolocate.miss') result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return { "location": { "lat": result['lat'], "lng": result['lon'], }, "accuracy": float(result['accuracy']), }
def test_empty(self): schema = self._make_schema() request = self._make_request('{}') data, errors = preprocess_request(request, schema, response=None) # missing lat and lon will default to -255 and be stripped out # instead of causing colander to drop the entire batch of # records self.assertEquals(data['lat'], None) self.assertEquals(data['lon'], None) self.assertFalse(errors)
def test_empty(self): schema = self._make_schema() request = self._make_request('{}') data, errors = preprocess_request(request, schema, response=None) # missing lat and lon will default to -255 and be stripped out # instead of causing colander to drop the entire batch of # records self.assertEquals(data['lat'], -255) self.assertEquals(data['lon'], -255) self.assertFalse(errors)
def preprocess(self): try: request_data, errors = preprocess_request( self.request, schema=self.schema(), response=self.error_response, ) except self.error_response: # capture JSON exceptions for submit calls self.raven_client.captureException() raise self.emit_upload_metrics(len(request_data['items'])) return request_data
def geosubmit_view(request): # Order matters here. We need to try the batch mode *before* the # single upload mode as classic w3c geolocate calls should behave # identically using either geosubmit or geolocate data, errors = preprocess_request( request, schema=GeoSubmitBatchSchema(), response=None, ) if any(data.get('items', ())): return process_batch(request, data, errors) else: return process_single(request)
def geosubmit_view(request): # Order matters here. We need to try the batch mode *before* the # single upload mode as classic w3c geolocate calls should behave # identically using either geosubmit or geolocate data, errors = preprocess_request( request, schema=GeoSubmitBatchSchema(), extra_checks=(geosubmit_validator,), response=None, ) if any(data.get('items', ())): return process_batch(request, data, errors) else: return process_single(request)
def search_view(request): api_key = request.GET.get('key', None) heka_client = get_heka_client() if api_key is None: # TODO: change into a better error response heka_client.incr('search.no_api_key') return {'status': 'not_found'} heka_client.incr('search.api_key') data, errors = preprocess_request( request, schema=SearchSchema(), extra_checks=(check_cell_or_wifi, ), ) session = request.db_slave_session result = None if data['wifi']: result = search_wifi(session, data) if result is not None: heka_client.incr('search.wifi_hit') if result is None: # no wifi result found, fall back to cell result = search_cell(session, data) if result is not None: heka_client.incr('search.cell_hit') if result is None and request.client_addr: # no cell or wifi, fall back again to geoip result = search_geoip(request.registry.geoip_db, request.client_addr) if result is not None: heka_client.incr('search.geoip_hit') if result is None: heka_client.incr('search.miss') return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def country_view(request): client_addr = request.client_addr geoip_db = request.registry.geoip_db if request.body in EMPTY and client_addr and geoip_db is not None: # Optimize common case of geoip-only request country = geoip_db.country_lookup(client_addr) if country: result = HTTPOk() result.content_type = 'application/json' result.text = '{"country_code": "%s", "country_name": "%s"}' % ( country.code, country.name) return result else: result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result data, errors = preprocess_request( request, schema=GeoLocateSchema(), response=JSONParseError, accept_empty=True, ) data = map_data(data, client_addr=client_addr) session = request.db_slave_session result = CountrySearcher( { 'geoip': geoip_db, 'session': session }, api_key_log=False, api_key_name=None, api_name='country', ).search(data) if not result: result = HTTPNotFound() result.content_type = 'application/json' result.body = NOT_FOUND return result return result
def search_view(request): heka_client = get_heka_client() data, errors = preprocess_request( request, schema=SearchSchema(), extra_checks=(check_cell_or_wifi, ), ) session = request.db_slave_session result = None if data['wifi']: result = search_wifi(session, data) if result is not None: heka_client.incr('search.wifi_hit') if result is None: # no wifi result found, fall back to cell result = search_cell(session, data) if result is not None: heka_client.incr('search.cell_hit') if result is None: # no direct cell result found, try cell LAC result = search_cell_lac(session, data) if result is not None: heka_client.incr('search.cell_lac_hit') if result is None and request.client_addr: # no cell or wifi, fall back again to geoip result = search_geoip(request.registry.geoip_db, request.client_addr) if result is not None: heka_client.incr('search.geoip_hit') if result is None: heka_client.incr('search.miss') return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def submit_view(request): data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): insert_measures.delay( items=dumps(items[i:i + 100]), nickname=nickname, ) return HTTPNoContent()
def submit_view(request): data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), ) items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): insert_measures.delay( # TODO convert items to json with support for decimal/datetime items=dumps(items[i:i + 100]), nickname=nickname, ) return HTTPNoContent()
def submit_view(request): try: data, errors = preprocess_request( request, schema=SubmitSchema(), extra_checks=(submit_validator, ), response=JSONError, ) except JSONError: # capture JSON exceptions for submit calls request.registry.heka_client.raven(RAVEN_ERROR) raise items = data['items'] nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') email = request.headers.get('X-Email', u'') if isinstance(email, str): email = email.decode('utf-8', 'ignore') # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, len(items), 100): batch = dumps(items[i:i + 100]) # insert measures, expire the task if it wasn't processed # after six hours to avoid queue overload try: insert_measures.apply_async( kwargs={ 'email': email, 'items': batch, 'nickname': nickname, }, expires=21600) except ConnectionError: # pragma: no cover return HTTPServiceUnavailable() return HTTPNoContent()
def country_view(request): request_data, errors = preprocess_request( request, schema=GeoLocateSchema(), response=JSONParseError, accept_empty=True, ) search_data = prepare_search_data(request_data, client_addr=request.client_addr) response = CountrySearcher( session_db=request.db_ro_session, geoip_db=request.registry.geoip_db, api_key=ApiKey(), api_name='country', ).search(search_data) if not response: response = HTTPNotFound() response.content_type = 'application/json' response.body = NOT_FOUND return response return response
def search_view(request): data, errors = preprocess_request( request, schema=SearchSchema(), accept_empty=True, ) session = request.db_slave_session result = search_all_sources( session, 'search', data, client_addr=request.client_addr, geoip_db=request.registry.geoip_db, api_key_log=getattr(request, 'api_key_log', False), api_key_name=getattr(request, 'api_key_name', None)) if not result: return {'status': 'not_found'} return { 'status': 'ok', 'lat': result['lat'], 'lon': result['lon'], 'accuracy': result['accuracy'], }
def test_empty(self): schema = self._make_schema() request = self._make_request('{}') data, errors = preprocess_request(request, schema, response=None) self.assertTrue(errors)
def test_empty_cell_entry(self): schema = self._make_schema() request = self._make_request('{"cell": [{}]}') data, errors = preprocess_request(request, schema, response=None) self.assertTrue('cell' in data)
def test_wrong_cell_data(self): schema = self._make_schema() request = self._make_request( '{"cell": [{"mcc": "a", "mnc": 2, "lac": 3, "cid": 4}]}') data, errors = preprocess_request(request, schema, response=None) self.assertTrue(errors)
def test_empty_wifi_entry(self): schema = self._make_schema() request = self._make_request( '{"lat": 12.3456781, "lon": 23.4567892, "wifi": [{}]}') data, errors = preprocess_request(request, schema, response=None) self.assertTrue(errors)