def test_invalid_radiotype(self): cell = CellFactory.build() cell2 = CellFactory.build(radio=Radio.wcdma) self.app.post_json( '/v1/geosubmit?key=test', {'items': [ {'latitude': cell.lat, 'longitude': cell.lon, 'cellTowers': [{ 'radioType': '18', 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, }, { 'radioType': 'umts', 'mobileCountryCode': cell2.mcc, 'mobileNetworkCode': cell2.mnc, 'locationAreaCode': cell2.lac, 'cellId': cell2.cid, }]}, ]}, status=200) obs = self.session.query(CellObservation).all() self.assertEqual(len(obs), 1) self.assertEqual(obs[0].cid, cell2.cid)
def test_invalid_radiotype(self): cell = CellFactory.build() cell2 = CellFactory.build(radio=Radio.wcdma) self.app.post_json('/v1/geosubmit?key=test', { 'items': [ { 'latitude': cell.lat, 'longitude': cell.lon, 'cellTowers': [{ 'radioType': '18', 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, }, { 'radioType': 'umts', 'mobileCountryCode': cell2.mcc, 'mobileNetworkCode': cell2.mnc, 'locationAreaCode': cell2.lac, 'cellId': cell2.cid, }] }, ] }, status=200) obs = self.session.query(CellObservation).all() self.assertEqual(len(obs), 1) self.assertEqual(obs[0].cid, cell2.cid)
def test_inconsistent_cell_radio_type_in_towers(self): cell = CellFactory(radio=Radio.umts, range=15000) cell2 = CellFactory(radio=Radio.gsm, range=35000, lat=cell.lat + 0.0002, lon=cell.lon) self.session.flush() res = self.app.post_json('%s?key=test' % self.url, { "radioType": Radio.cdma.name, "cellTowers": [ { "radio": "cdma", "radioType": "wcdma", "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, { "radioType": cell2.radio.name, "mobileCountryCode": cell2.mcc, "mobileNetworkCode": cell2.mnc, "locationAreaCode": cell2.lac, "cellId": cell2.cid }, ] }, status=200) location = res.json['location'] self.assertAlmostEquals(location['lat'], cell.lat) self.assertAlmostEquals(location['lng'], cell.lon) self.assertEqual(res.json['accuracy'], cell.range)
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2, api_key='test', email=None, ip=None, nickname=None, blue_key=None, cell_mcc=None, wifi_key=None, lat=None, lon=None): reports = [] for i in range(num): pos = CellFactory.build() report = { 'timestamp': time.time() * 1000.0, 'position': {}, 'bluetoothBeacons': [], 'cellTowers': [], 'wifiAccessPoints': [], } report['position']['latitude'] = lat or pos.lat report['position']['longitude'] = lon or pos.lon report['position']['accuracy'] = 17.0 + i blues = WifiShardFactory.build_batch(blue_factor, lat=pos.lat, lon=pos.lon) for blue in blues: blue_data = { 'macAddress': blue_key or blue.mac, 'signalStrength': -100 + i, } report['bluetoothBeacons'].append(blue_data) cells = CellFactory.build_batch(cell_factor, lat=pos.lat, lon=pos.lon) for cell in cells: cell_data = { 'radioType': cell.radio.name, 'mobileCountryCode': cell_mcc or cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': cell.psc, 'signalStrength': -110 + i, } report['cellTowers'].append(cell_data) wifis = WifiShardFactory.build_batch(wifi_factor, lat=pos.lat, lon=pos.lon) for wifi in wifis: wifi_data = { 'macAddress': wifi_key or wifi.mac, 'signalStrength': -90 + i, 'ssid': 'my-wifi', } report['wifiAccessPoints'].append(wifi_data) reports.append(report) queue_reports.delay(reports=reports, api_key=api_key, email=email, ip=ip, nickname=nickname).get() return reports
def test_export_full(self): CellFactory.create_batch(10, radio=Radio.gsm) self.session.commit() with mock_s3() as mock_key: cell_export_full(_bucket='localhost.bucket') pat = r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz' self.assertRegex(mock_key.key, pat) method = mock_key.set_contents_from_filename self.assertRegex(method.call_args[0][0], pat)
def test_daily_export(self): CellFactory.create_batch(10, radio=Radio.gsm) self.session.commit() with mock_s3() as mock_key: export_modified_cells(bucket='localhost.bucket', hourly=False) pat = r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz' self.assertRegexpMatches(mock_key.key, pat) method = mock_key.set_contents_from_filename self.assertRegexpMatches(method.call_args[0][0], pat)
def test_cdma_cell(self): # Specifying a CDMA radio type works, # but the information is ignored. cell = CellFactory(radio=Radio.gsm, radius=15000) cell2 = CellFactory(radio=Radio.gsm, radius=35000, lat=cell.lat + 0.0002, lon=cell.lon) cell2.radio = Radio.cdma self.session.flush() query = self.model_query(cells=[cell, cell2]) res = self._call(body=query) self.check_model_response(res, cell)
def test_cell(self): cell = CellFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) self.assertEqual(len(query.cell), 1) self.assertEqual(query.expected_accuracy, DataAccuracy.medium) query_cell = query.cell[0] for key, value in cell_query[0].items(): query_value = getattr(query_cell, key, None) if key == 'radio': self.assertEqual(query_value, cell.radio) else: self.assertEqual(query_value, value) self.assertEqual(len(query.cell_area), 1) query_area = query.cell_area[0] for key, value in cell_query[0].items(): query_value = getattr(query_area, key, None) if key == 'radio': self.assertEqual(query_value, cell.radio) elif key in ('cid', 'psc'): pass else: self.assertEqual(query_value, value)
def test_ok_partial_cell(self): session = self.session cell = CellFactory() session.flush() res = self.app.post_json('%s?key=test' % self.url, { "cellTowers": [{ "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "psc": cell.psc }, { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "psc": cell.psc + 1, }] }, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, { "location": { "lat": cell.lat, "lng": cell.lon }, "accuracy": cell.range })
def test_ok_cell_radio_in_celltowers_dupes(self): # This test covers a bug related to FxOS calling the # geolocate API incorrectly. cell = CellFactory() self.session.flush() res = self.app.post_json('%s?key=test' % self.url, { "cellTowers": [ { "radio": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, { "radio": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, ] }, status=200) self.assertEqual(res.json, { "location": { "lat": cell.lat, "lng": cell.lon }, "accuracy": cell.range })
def test_ok_cell_radiotype_in_celltowers(self): # This test covers an extension to the geolocate API cell = CellFactory() self.session.flush() res = self.app.post_json('%s?key=test' % self.url, { "cellTowers": [ { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, ] }, status=200) self.assertEqual(res.json, { "location": { "lat": cell.lat, "lng": cell.lon }, "accuracy": cell.range })
def test_ok_cell(self): cell = CellFactory() self.session.flush() res = self.app.post_json('%s?key=test' % self.url, { "radioType": cell.radio.name, "cellTowers": [ { "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, ] }, status=200) self.check_stats(counter=[ self.metric_url + '.200', self.metric + '.api_key.test', self.metric + '.api_log.test.cell_hit' ]) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, { "location": { "lat": cell.lat, "lng": cell.lon }, "accuracy": cell.range })
def test_cell_mcc_mnc_strings(self): # mcc and mnc are officially defined as strings, where "01" is # different from "1". In practice many systems ours included treat # them as integers, so both of these are encoded as 1 instead. # Some clients sends us these values as strings, some as integers, # so we want to make sure we support both. cell = CellFactory(mnc=1) self.session.flush() res = self.app.post_json('%s?key=test' % self.url, { "cellTowers": [ { "radioType": cell.radio.name, "mobileCountryCode": str(cell.mcc), "mobileNetworkCode": "01", "locationAreaCode": cell.lac, "cellId": cell.cid }, ] }, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, { "location": { "lat": cell.lat, "lng": cell.lon }, "accuracy": cell.range })
def test_database_error(self, db_errors=0): cells = CellFactory.build_batch(2) wifis = WifiShardFactory.build_batch(2) for model in (Cell, CellArea, CellOCID, CellAreaOCID): self.session.execute(text('drop table %s;' % model.__tablename__)) for name in set([wifi.__tablename__ for wifi in wifis]): self.session.execute(text('drop table %s;' % name)) query = self.model_query(cells=cells, wifis=wifis) res = self._call(body=query, ip=self.test_ip) self.check_response(res, 'ok') self.check_stats(counter=[ ('request', [self.metric_path, 'method:post', 'status:200']), ], timer=[ ('request', [self.metric_path, 'method:post']), ]) if self.apikey_metrics: self.check_stats(counter=[ (self.metric_type + '.result', ['key:test', 'region:GB', 'fallback_allowed:false', 'accuracy:high', 'status:miss']), ]) self.check_raven([('ProgrammingError', db_errors)])
def test_medium_hit(self): cells = CellFactory.build_batch(1) self._make_query(self._make_result(accuracy=30000.0), cell=cells) self.check_stats(counter=[ ('locate.result', ['key:key', 'country:none', 'accuracy:medium', 'status:hit']), ])
def test_lte_radio(self): cell = CellFactory(radio=Radio.lte) self.session.flush() res = self.app.post_json('%s?key=test' % self.url, { "cellTowers": [ { "radio": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, ] }, status=200) self.check_stats( counter=[self.metric_url + '.200', self.metric + '.api_key.test']) self.assertEqual(res.content_type, 'application/json') location = res.json['location'] self.assertAlmostEquals(location['lat'], cell.lat) self.assertAlmostEquals(location['lng'], cell.lon) self.assertEqual(res.json['accuracy'], cell.range)
def test_database_error(self): london = self.geoip_data['London'] self.session.execute(text('drop table wifi;')) self.session.execute(text('drop table cell;')) cell = CellFactory.build() wifis = WifiFactory.build_batch(2) res = self.app.post_json( '/v1/geolocate?key=test', { 'cellTowers': [{ 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid}, ], 'wifiAccessPoints': [ {'macAddress': wifis[0].key}, {'macAddress': wifis[1].key}, ]}, extra_environ={'HTTP_X_FORWARDED_FOR': london['ip']}, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {'location': {'lat': london['latitude'], 'lng': london['longitude']}, 'accuracy': london['accuracy']}) self.check_stats( timer=['request.v1.geolocate'], counter=[ 'request.v1.geolocate.200', 'geolocate.geoip_hit', ]) self.check_raven([('ProgrammingError', 2)])
def test_cell(self): # cell with unique mcc to region mapping cell = CellFactory.create(mcc=235) query = self.model_query(cells=[cell]) res = self._call(body=query) self.check_model_response(res, cell, region='GB') self.check_db_calls(rw=0, ro=0)
def test_locate_finds_country_from_mcc(self): country = mobile_codes.mcc("235")[0] cell = CellFactory.build(mcc=235) query = self.model_query(cells=[cell]) location = self.provider.locate(query) self.check_model_location(location, country)
def test_empty_result_from_fallback_cached(self): cell = CellFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=LocationNotFound.json_body(), status_code=404 ) query = self.model_query(cells=[cell]) location = self.provider.locate(query) self.check_model_location(location, None) self.assertEqual(mock_request.call_count, 1) self.check_stats( counter=[ 'm.fallback.lookup_status.404', 'm.fallback.cache.miss', ], timer=['m.fallback.lookup']) query = self.model_query(cells=[cell]) location = self.provider.locate(query) self.check_model_location(location, None) self.assertEqual(mock_request.call_count, 1) self.check_stats( counter=[ 'm.fallback.lookup_status.404', 'm.fallback.cache.hit', ], timer=['m.fallback.lookup'])
def test_fallback_used_when_geoip_also_present(self): cells = CellFactory.build_batch(2, radio=Radio.wcdma) wifis = WifiFactory.build_batch(3) api_key = ApiKey.getkey(self.session, 'test') api_key.allow_fallback = True self.session.flush() with requests_mock.Mocker() as mock: response_location = { 'location': { 'lat': 1.0, 'lng': 1.0, }, 'accuracy': 100, } mock.register_uri( 'POST', requests_mock.ANY, json=response_location) query = self.model_query(cells=cells, wifis=wifis) res = self._call(body=query, ip=self.test_ip) send_json = mock.request_history[0].json() self.assertEqual(len(send_json['cellTowers']), 2) self.assertEqual(len(send_json['wifiAccessPoints']), 3) self.check_model_response(res, None, lat=1.0, lon=1.0, accuracy=100) self.check_stats( timer=[self.metric_url], counter=[self.metric + '.api_key.test', self.metric + '.fallback_hit', self.metric_url + '.200', self.metric + '.api_log.test.fallback_hit'], )
def test_ok_no_existing_cell(self): session = self.session now_ms = int(time.time() * 1000) first_of_month = utcnow().replace(day=1, hour=0, minute=0, second=0) cell = CellFactory.build() res = self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": cell.lat, "longitude": cell.lon, "accuracy": 12.4, "altitude": 100.1, "altitudeAccuracy": 23.7, "heading": 45.0, "speed": 3.6, "timestamp": now_ms, "cellTowers": [{ "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "age": 3, "asu": 31, "psc": cell.psc, "signalStrength": -51, "timingAdvance": 1, }] }, ] }, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {}) self.assertEquals(session.query(Cell).count(), 1) result = session.query(CellObservation).all() self.assertEquals(len(result), 1) obs = result[0] for name in ('lat', 'lon', 'radio', 'mcc', 'mnc', 'lac', 'cid', 'psc'): self.assertEqual(getattr(obs, name), getattr(cell, name)) self.assertEqual(obs.accuracy, 12) self.assertEqual(obs.altitude, 100) self.assertEqual(obs.altitude_accuracy, 24) self.assertEqual(obs.heading, 45.0) self.assertEqual(obs.speed, 3.6) self.assertEqual(obs.time, first_of_month) self.assertEqual(obs.asu, 31) self.assertEqual(obs.signal, -51) self.assertEqual(obs.ta, 1)
def test_duplicated_cell_observations(self): session = self.session cell = CellFactory.build() self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": cell.lat, "longitude": cell.lon, "cellTowers": [ { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "asu": 10 }, { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "asu": 16 }, ] }, ] }, status=200) self.assertEquals(session.query(CellObservation).count(), 1)
def test_single_cell_results_cached_preventing_external_call(self): cell = CellFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_location) query = self.model_query(cells=[cell]) query.cell[0]['signal'] = -77 location = self.provider.locate(query) self.check_model_location(location, self.fallback_model) self.assertEqual(mock_request.call_count, 1) self.check_stats( counter=[ 'm.fallback.lookup_status.200', 'm.fallback.cache.miss', ], timer=['m.fallback.lookup']) # vary the signal strength, not part of cache key query.cell[0]['signal'] = -82 location = self.provider.locate(query) self.check_model_location(location, self.fallback_model) self.assertEqual(mock_request.call_count, 1) self.check_stats( counter=[ 'm.fallback.lookup_status.200', 'm.fallback.cache.hit', ], timer=['m.fallback.lookup'])
def test_cell_geoip_mismatch(self): # UK GeoIP with US mcc cell = CellFactory.create(mcc=310) query = self.model_query(cells=[cell]) res = self._call(body=query, ip=self.test_ip) self.check_model_response(res, cell, country='US') self.check_db_calls(rw=0, ro=0)
def test_no_call_made_when_not_allowed_for_apikey(self): cells = CellFactory.build_batch(2) wifis = WifiFactory.build_batch(2) self.provider.api_key.allow_fallback = False query = self.model_query(cells=cells, wifis=wifis) self.check_should_locate(query, False)
def test_cache_single_cell(self): cell = CellFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query(cells=[cell]) query.cell[0].signal = -77 result = self.source.search(query) self.check_model_result(result, self.fallback_model) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ('locate.fallback.lookup', ['status:200']), ], timer=[ 'locate.fallback.lookup', ]) # vary the signal strength, not part of cache key query.cell[0].signal = -82 result = self.source.search(query) self.check_model_result(result, self.fallback_model) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:hit']), ('locate.fallback.lookup', ['status:200']), ], timer=[ 'locate.fallback.lookup', ])
def test_cache_empty_result(self): cell = CellFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=LocationNotFound.json_body(), status_code=404 ) query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, None) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ('locate.fallback.lookup', ['status:404']), ]) query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, None) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:hit']), ('locate.fallback.lookup', ['status:404']), ])
def test_set_cache_redis_failure(self): cell = CellFactory.build() mock_redis_client = self._mock_redis_client() mock_redis_client.mget.return_value = [] mock_redis_client.mset.side_effect = RedisError() mock_redis_client.expire.side_effect = RedisError() mock_redis_client.execute.side_effect = RedisError() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) with mock.patch.object(self.source.cache, 'redis_client', mock_redis_client): query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, self.fallback_model) self.assertTrue(mock_redis_client.mget.called) self.assertTrue(mock_redis_client.mset.called) self.assertTrue(mock_request.called) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ])
def test_api_key_disallows(self): api_key = ApiKeyFactory.build(allow_fallback=False) cells = CellFactory.build_batch(2) wifis = WifiShardFactory.build_batch(2) query = self.model_query(cells=cells, wifis=wifis, api_key=api_key) self.check_should_search(query, False)
def test_success(self): cell = CellFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query( cells=[cell], fallback={ 'lacf': True, 'ipf': False, }, ) result = self.source.search(query) self.check_model_result(result, self.fallback_model) request_json = mock_request.request_history[0].json() self.assertEqual(request_json['fallbacks'], {'lacf': True}) self.check_stats(counter=[ ('locate.fallback.lookup', ['status:200']), ], timer=[ 'locate.fallback.lookup', ])
def test_database_error(self, db_errors=0): for tablename in ('cell', 'cell_area', 'ocid_cell', 'ocid_cell_area'): self.session.execute(text('drop table %s;' % tablename)) for i in range(16): self.session.execute(text( 'drop table wifi_shard_%s;' % hex(i)[2:])) cells = CellFactory.build_batch(2) wifis = WifiShardFactory.build_batch(2) query = self.model_query(cells=cells, wifis=wifis) res = self._call(body=query, ip=self.test_ip) self.check_response(res, 'ok') self.check_stats(counter=[ ('request', [self.metric_path, 'method:post', 'status:200']), ], timer=[ ('request', [self.metric_path, 'method:post']), ]) if self.apikey_metrics: self.check_stats(counter=[ (self.metric_type + '.result', ['key:test', 'country:GB', 'accuracy:high', 'status:miss']), ]) self.check_raven([('ProgrammingError', db_errors)])
def test_duplicated_cell_observations(self): session = self.session cell = CellFactory.build() self.app.post_json( '/v1/geosubmit?key=test', {"items": [ {"latitude": cell.lat, "longitude": cell.lon, "cellTowers": [ {"radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "asu": 10}, {"radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "asu": 16}, ]}, ]}, status=200) self.assertEquals(session.query(CellObservation).count(), 1)
def test_cell(self): cell = CellFactory.build() cell_query = { 'radio': cell.radio.name, 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'signal': -90, 'ta': 1, } query = Query(cell=[cell_query]) self.assertEqual(len(query.cell), 1) query_cell = query.cell[0] for key, value in cell_query.items(): if key == 'radio': self.assertEqual(query_cell[key], cell.radio) else: self.assertEqual(query_cell[key], value) self.assertEqual(len(query.cell_area), 1) query_area = query.cell_area[0] for key, value in cell_query.items(): if key == 'radio': self.assertEqual(query_area[key], cell.radio) elif key in ('cid', 'psc'): pass else: self.assertEqual(query_area[key], value)
def test_cell_ambiguous(self): # cell with ambiguous mcc to country mapping cell = CellFactory.create(mcc=234) query = self.model_query(cells=[cell]) res = self._call(body=query) self.check_model_response(res, cell, country='GB') self.check_db_calls(rw=0, ro=0)
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) cell = CellFactory.build() self.session.add( CellBlacklist(time=last_week, count=1, radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid)) self.session.flush() # add a new entry for the previously blacklisted cell obs = dict(lat=cell.lat, lon=cell.lon, radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid) insert_measures_cell.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def add_reports(self, number=3, api_key='test', email=None): reports = [] for i in range(number): report = { 'timestamp': time.time() * 1000.0, 'position': {}, 'cellTowers': [], 'wifiAccessPoints': [], } cell = CellFactory.build() report['position']['latitude'] = cell.lat report['position']['longitude'] = cell.lon report['position']['accuracy'] = 17 + i cell_data = { 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': cell.psc, 'signalStrength': -110 + i, } report['cellTowers'].append(cell_data) wifis = WifiFactory.build_batch(2, lat=cell.lat, lon=cell.lon) for wifi in wifis: wifi_data = { 'macAddress': wifi.key, 'signalStrength': -90 + i, } report['wifiAccessPoints'].append(wifi_data) reports.append(report) queue_reports.delay(reports=reports, api_key=api_key, email=email).get() return reports
def setUp(self): super(TestFallbackProvider, self).setUp() self.provider.api_key.allow_fallback = True self.response_location = { 'location': { 'lat': 51.5366, 'lng': 0.03989, }, 'accuracy': 1500, 'fallback': 'lacf', } self.cells = [] for cell in CellFactory.build_batch(2): self.cells.append({ 'radio': cell.radio, 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, 'signal': -70, }) self.cells[0]['ta'] = 1 self.wifis = [] for wifi in WifiFactory.build_batch(2): self.wifis.append({ 'key': wifi.key, 'signal': -77, }) self.wifis[0]['channel'] = 6 self.wifis[0]['frequency'] = 2437 self.wifis[0]['snr'] = 13
def test_blacklist(self): now = util.utcnow() cell = CellFactory.build() observations = [dict(radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid + i, psc=cell.psc, lat=cell.lat + i * 0.0000001, lon=cell.lon + i * 0.0000001) for i in range(1, 4)] black = CellBlacklist( radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid + 1, time=now, count=1, ) self.session.add(black) self.session.flush() result = insert_measures_cell.delay(observations) self.assertEqual(result.get(), 2) self.assertEqual(self.data_queue.size(), 2) update_cell.delay().get() cells = self.session.query(Cell).all() self.assertEqual(len(cells), 2) self.check_statcounter(StatKey.cell, 2) self.check_statcounter(StatKey.unique_cell, 2)
def add_reports(self, number=3, api_key='test', email=None): reports = [] for i in range(number): report = { 'timestamp': time.time() * 1000.0, 'position': {}, 'cellTowers': [], 'wifiAccessPoints': [], } cell = CellFactory.build() report['position']['latitude'] = cell.lat report['position']['longitude'] = cell.lon report['position']['accuracy'] = 17 + i cell_data = { 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': cell.psc, 'signalStrength': -110 + i, } report['cellTowers'].append(cell_data) wifis = WifiFactory.build_batch(2, lat=cell.lat, lon=cell.lon) for wifi in wifis: wifi_data = { 'macAddress': wifi.key, 'signalStrength': -90 + i, } report['wifiAccessPoints'].append(wifi_data) reports.append(report) queue_reports.delay( reports=reports, api_key=api_key, email=email).get() return reports
def test_get_cell_multi(self): cells = CellFactory.build_batch(2) query = Query(cell=self.cell_model_query(cells)) self.assertEqual(self.cache.get(query), None) self.check_stats(counter=[ ('locate.fallback.cache', 1, 1, ['status:bypassed']), ])
def test_unique_cell_histogram(self): session = self.session today = util.utcnow() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(3)) CellFactory(created=long_ago, radio=Radio.gsm) CellFactory(created=two_days, radio=Radio.umts) CellFactory(created=two_days, radio=Radio.umts, cid=50) CellFactory(created=yesterday, radio=Radio.lte, cid=50) CellFactory(created=today, radio=Radio.gsm, mnc=30) session.flush() result = unique_cell_histogram.delay(ago=3) self.assertEqual(result.get(), 1) stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, StatKey.unique_cell) self.assertEqual(stats[0].time, long_ago.date()) self.assertEqual(stats[0].value, 1) # fill up newer dates unique_cell_histogram.delay(ago=2).get() unique_cell_histogram.delay(ago=1).get() unique_cell_histogram.delay(ago=0).get() # test duplicate execution unique_cell_histogram.delay(ago=1).get() stats = session.query(Stat.time, Stat.value).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(dict(stats), { long_ago.date(): 1, two_days.date(): 3, yesterday.date(): 4, today.date(): 5})
def test_database_error(self): london = self.geoip_data['London'] session = self.session stmt = text("drop table wifi;") session.execute(stmt) stmt = text("drop table cell;") session.execute(stmt) cell = CellFactory.build() wifis = WifiFactory.build_batch(2) res = self.app.post_json( '/v1/geolocate?key=test', { "cellTowers": [ { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, ], "wifiAccessPoints": [ { "macAddress": wifis[0].key }, { "macAddress": wifis[1].key }, ] }, extra_environ={'HTTP_X_FORWARDED_FOR': london['ip']}, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual( res.json, { "location": { "lat": london['latitude'], "lng": london['longitude'] }, "accuracy": london['accuracy'] }) self.check_stats(timer=['request.v1.geolocate'], counter=[ 'request.v1.geolocate.200', 'geolocate.geoip_hit', ]) self.check_raven([('ProgrammingError', 2)])
def test_missing_radio(self): cell = CellFactory.build() self.app.post_json('/v1/submit', { 'items': [{ 'lat': cell.lat, 'lon': cell.lon, 'cell': [{ 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, }] }] }, status=204) self.assertEqual(self.session.query(CellObservation).count(), 0)
def test_no_api_key(self): cell = CellFactory() self.session.flush() res = self.app.post_json(self.url, { "cellTowers": [ { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid }, ] }, status=400) self.assertEqual(res.content_type, 'application/json') self.assertEqual(u'Invalid API key', res.json['error']['message']) self.check_stats(counter=[self.metric + '.no_api_key'])
def test_invalid_cell(self): cell = CellFactory.build() self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": cell.lat, "longitude": cell.lon, "cellTowers": [{ "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": 2000, "locationAreaCode": cell.lac, "cellId": cell.cid, }] }, ] }, status=200) self.assertEquals(self.session.query(CellObservation).count(), 0)
def test_ok_partial_cell(self): session = self.session cell = CellFactory() session.flush() res = self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": cell.lat, "longitude": cell.lon, "cellTowers": [{ "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "psc": cell.psc }, { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "psc": cell.psc + 1, }] }, ] }, status=200) # check that we get an empty response self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {}) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 2) pscs = set([obs.psc for obs in observations]) self.assertEqual(pscs, set([cell.psc, cell.psc + 1]))
def test_ok_radioless_cell(self): cell = CellFactory.build() self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": cell.lat, "longitude": cell.lon, "cellTowers": [{ "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "psc": cell.psc, }] }, ] }, status=200) observations = self.session.query(CellObservation).all() self.assertEqual(len(observations), 0)
def test_ok_cell(self): session = self.session cell = CellFactory() new_cell = CellFactory.build() session.flush() res = self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": cell.lat, "longitude": cell.lon, "radioType": cell.radio.name, "cellTowers": [{ "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, }] }, { "latitude": new_cell.lat, "longitude": new_cell.lon, "cellTowers": [{ "radioType": new_cell.radio.name, "mobileCountryCode": new_cell.mcc, "mobileNetworkCode": new_cell.mnc, "locationAreaCode": new_cell.lac, "cellId": new_cell.cid, }] }, ] }, status=200) # check that we get an empty response self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {}) self.assertEqual(session.query(Cell).count(), 2) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 2) radios = set([obs.radio for obs in observations]) self.assertEqual(radios, set([cell.radio, new_cell.radio])) self.check_stats(counter=[ 'geosubmit.api_key.test', 'items.api_log.test.uploaded.batches', 'items.api_log.test.uploaded.reports', 'items.api_log.test.uploaded.cell_observations', 'items.uploaded.cell_observations', 'items.uploaded.batches', 'items.uploaded.reports', 'request.v1.geosubmit.200', ], timer=[ 'items.api_log.test.uploaded.batch_size', 'items.uploaded.batch_size', 'request.v1.geosubmit' ])
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(hours=1) schema = ValidCellKeySchema() obs_factory = CellObservationFactory invalid_key = dict(lac=schema.fields['lac'].missing, cid=schema.fields['cid'].missing) cell1 = CellFactory(new_measures=3, total_measures=5) lat1, lon1 = (cell1.lat, cell1.lon) key1 = dict(lac=cell1.lac, cid=cell1.cid) obs_factory(lat=lat1, lon=lon1, created=now, **key1) obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1) obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1) # The lac, cid are invalid and should be skipped obs_factory.create_batch(2, created=now, **invalid_key) cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, new_measures=2, total_measures=4) lat2, lon2 = (cell2.lat, cell2.lon) key2 = dict(lac=cell2.lac, cid=cell2.cid) # the lat/lon is bogus and mismatches the line above on purpose # to make sure old observations are skipped obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2) obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2) obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2) obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2) cell3 = CellFactory(new_measures=10, total_measures=100000) lat3, lon3 = (cell3.lat, cell3.lon) obs_factory.create_batch(10, lat=lat3 + 1.0, lon=lon3 + 1.0, **dict(lac=cell3.lac, cid=cell3.cid)) self.session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (3, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 3) self.assertEqual(set([c.new_measures for c in cells]), set([0])) for cell in cells: if cell.hashkey() == cell1.hashkey(): self.assertEqual(cell.lat, lat1 + 0.002) self.assertEqual(cell.lon, lon1 + 0.003) if cell.hashkey() == cell2.hashkey(): self.assertEqual(cell.lat, lat2 + 0.001) self.assertEqual(cell.lon, lon2 + 0.002) if cell.hashkey() == cell3.hashkey(): expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010 expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010 self.assertAlmostEqual(cell.lat, expected_lat, 7) self.assertAlmostEqual(cell.lon, expected_lon, 7)