def test_invalid_radiotype(self):
cell = CellFactory.build()
cell2 = CellFactory.build(radio=Radio.wcdma)
self.app.post_json(
'/v1/geosubmit?key=test',
{'items': [
{'latitude': cell.lat,
'longitude': cell.lon,
'cellTowers': [{
'radioType': '18',
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
}, {
'radioType': 'umts',
'mobileCountryCode': cell2.mcc,
'mobileNetworkCode': cell2.mnc,
'locationAreaCode': cell2.lac,
'cellId': cell2.cid,
}]},
]},
status=200)
obs = self.session.query(CellObservation).all()
self.assertEqual(len(obs), 1)
self.assertEqual(obs[0].cid, cell2.cid)
def test_invalid_radiotype(self):
cell = CellFactory.build()
cell2 = CellFactory.build(radio=Radio.wcdma)
self.app.post_json('/v1/geosubmit?key=test', {
'items': [
{
'latitude':
cell.lat,
'longitude':
cell.lon,
'cellTowers': [{
'radioType': '18',
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
}, {
'radioType': 'umts',
'mobileCountryCode': cell2.mcc,
'mobileNetworkCode': cell2.mnc,
'locationAreaCode': cell2.lac,
'cellId': cell2.cid,
}]
},
]
},
status=200)
obs = self.session.query(CellObservation).all()
self.assertEqual(len(obs), 1)
self.assertEqual(obs[0].cid, cell2.cid)
def test_inconsistent_cell_radio_type_in_towers(self):
cell = CellFactory(radio=Radio.umts, range=15000)
cell2 = CellFactory(radio=Radio.gsm,
range=35000,
lat=cell.lat + 0.0002,
lon=cell.lon)
self.session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"radioType":
Radio.cdma.name,
"cellTowers": [
{
"radio": "cdma",
"radioType": "wcdma",
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
{
"radioType": cell2.radio.name,
"mobileCountryCode": cell2.mcc,
"mobileNetworkCode": cell2.mnc,
"locationAreaCode": cell2.lac,
"cellId": cell2.cid
},
]
},
status=200)
location = res.json['location']
self.assertAlmostEquals(location['lat'], cell.lat)
self.assertAlmostEquals(location['lng'], cell.lon)
self.assertEqual(res.json['accuracy'], cell.range)
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', email=None, ip=None, nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = WifiShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
queue_reports.delay(reports=reports, api_key=api_key,
email=email, ip=ip, nickname=nickname).get()
return reports
def test_export_full(self):
CellFactory.create_batch(10, radio=Radio.gsm)
self.session.commit()
with mock_s3() as mock_key:
cell_export_full(_bucket='localhost.bucket')
pat = r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz'
self.assertRegex(mock_key.key, pat)
method = mock_key.set_contents_from_filename
self.assertRegex(method.call_args[0][0], pat)
def test_daily_export(self):
CellFactory.create_batch(10, radio=Radio.gsm)
self.session.commit()
with mock_s3() as mock_key:
export_modified_cells(bucket='localhost.bucket', hourly=False)
pat = r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz'
self.assertRegexpMatches(mock_key.key, pat)
method = mock_key.set_contents_from_filename
self.assertRegexpMatches(method.call_args[0][0], pat)
def test_cdma_cell(self):
# Specifying a CDMA radio type works,
# but the information is ignored.
cell = CellFactory(radio=Radio.gsm, radius=15000)
cell2 = CellFactory(radio=Radio.gsm, radius=35000,
lat=cell.lat + 0.0002, lon=cell.lon)
cell2.radio = Radio.cdma
self.session.flush()
query = self.model_query(cells=[cell, cell2])
res = self._call(body=query)
self.check_model_response(res, cell)
def test_cell(self):
cell = CellFactory.build()
cell_query = self.cell_model_query([cell])
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 1)
self.assertEqual(query.expected_accuracy, DataAccuracy.medium)
query_cell = query.cell[0]
for key, value in cell_query[0].items():
query_value = getattr(query_cell, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
else:
self.assertEqual(query_value, value)
self.assertEqual(len(query.cell_area), 1)
query_area = query.cell_area[0]
for key, value in cell_query[0].items():
query_value = getattr(query_area, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
elif key in ('cid', 'psc'):
pass
else:
self.assertEqual(query_value, value)
def test_ok_partial_cell(self):
session = self.session
cell = CellFactory()
session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"cellTowers": [{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"psc": cell.psc
}, {
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"psc": cell.psc + 1,
}]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {
"location": {
"lat": cell.lat,
"lng": cell.lon
},
"accuracy": cell.range
})
def test_ok_cell_radio_in_celltowers_dupes(self):
# This test covers a bug related to FxOS calling the
# geolocate API incorrectly.
cell = CellFactory()
self.session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"cellTowers": [
{
"radio": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
{
"radio": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
]
},
status=200)
self.assertEqual(res.json, {
"location": {
"lat": cell.lat,
"lng": cell.lon
},
"accuracy": cell.range
})
def test_ok_cell_radiotype_in_celltowers(self):
# This test covers an extension to the geolocate API
cell = CellFactory()
self.session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"cellTowers": [
{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
]
},
status=200)
self.assertEqual(res.json, {
"location": {
"lat": cell.lat,
"lng": cell.lon
},
"accuracy": cell.range
})
def test_ok_cell(self):
cell = CellFactory()
self.session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"radioType":
cell.radio.name,
"cellTowers": [
{
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
]
},
status=200)
self.check_stats(counter=[
self.metric_url + '.200', self.metric +
'.api_key.test', self.metric + '.api_log.test.cell_hit'
])
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {
"location": {
"lat": cell.lat,
"lng": cell.lon
},
"accuracy": cell.range
})
def test_cell_mcc_mnc_strings(self):
# mcc and mnc are officially defined as strings, where "01" is
# different from "1". In practice many systems ours included treat
# them as integers, so both of these are encoded as 1 instead.
# Some clients sends us these values as strings, some as integers,
# so we want to make sure we support both.
cell = CellFactory(mnc=1)
self.session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"cellTowers": [
{
"radioType": cell.radio.name,
"mobileCountryCode": str(cell.mcc),
"mobileNetworkCode": "01",
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {
"location": {
"lat": cell.lat,
"lng": cell.lon
},
"accuracy": cell.range
})
def test_database_error(self, db_errors=0):
cells = CellFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
for model in (Cell, CellArea, CellOCID, CellAreaOCID):
self.session.execute(text('drop table %s;' % model.__tablename__))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text('drop table %s;' % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
self.check_stats(counter=[
(self.metric_type + '.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
self.check_raven([('ProgrammingError', db_errors)])
def test_medium_hit(self):
cells = CellFactory.build_batch(1)
self._make_query(self._make_result(accuracy=30000.0), cell=cells)
self.check_stats(counter=[
('locate.result',
['key:key', 'country:none', 'accuracy:medium', 'status:hit']),
])
def test_lte_radio(self):
cell = CellFactory(radio=Radio.lte)
self.session.flush()
res = self.app.post_json('%s?key=test' % self.url, {
"cellTowers": [
{
"radio": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
]
},
status=200)
self.check_stats(
counter=[self.metric_url + '.200', self.metric + '.api_key.test'])
self.assertEqual(res.content_type, 'application/json')
location = res.json['location']
self.assertAlmostEquals(location['lat'], cell.lat)
self.assertAlmostEquals(location['lng'], cell.lon)
self.assertEqual(res.json['accuracy'], cell.range)
def test_database_error(self):
london = self.geoip_data['London']
self.session.execute(text('drop table wifi;'))
self.session.execute(text('drop table cell;'))
cell = CellFactory.build()
wifis = WifiFactory.build_batch(2)
res = self.app.post_json(
'/v1/geolocate?key=test', {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid},
],
'wifiAccessPoints': [
{'macAddress': wifis[0].key},
{'macAddress': wifis[1].key},
]},
extra_environ={'HTTP_X_FORWARDED_FOR': london['ip']},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {'location': {'lat': london['latitude'],
'lng': london['longitude']},
'accuracy': london['accuracy']})
self.check_stats(
timer=['request.v1.geolocate'],
counter=[
'request.v1.geolocate.200',
'geolocate.geoip_hit',
])
self.check_raven([('ProgrammingError', 2)])
def test_cell(self):
# cell with unique mcc to region mapping
cell = CellFactory.create(mcc=235)
query = self.model_query(cells=[cell])
res = self._call(body=query)
self.check_model_response(res, cell, region='GB')
self.check_db_calls(rw=0, ro=0)
def test_locate_finds_country_from_mcc(self):
country = mobile_codes.mcc("235")[0]
cell = CellFactory.build(mcc=235)
query = self.model_query(cells=[cell])
location = self.provider.locate(query)
self.check_model_location(location, country)
def test_empty_result_from_fallback_cached(self):
cell = CellFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST',
requests_mock.ANY,
json=LocationNotFound.json_body(),
status_code=404
)
query = self.model_query(cells=[cell])
location = self.provider.locate(query)
self.check_model_location(location, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(
counter=[
'm.fallback.lookup_status.404',
'm.fallback.cache.miss',
],
timer=['m.fallback.lookup'])
query = self.model_query(cells=[cell])
location = self.provider.locate(query)
self.check_model_location(location, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(
counter=[
'm.fallback.lookup_status.404',
'm.fallback.cache.hit',
],
timer=['m.fallback.lookup'])
def test_fallback_used_when_geoip_also_present(self):
cells = CellFactory.build_batch(2, radio=Radio.wcdma)
wifis = WifiFactory.build_batch(3)
api_key = ApiKey.getkey(self.session, 'test')
api_key.allow_fallback = True
self.session.flush()
with requests_mock.Mocker() as mock:
response_location = {
'location': {
'lat': 1.0,
'lng': 1.0,
},
'accuracy': 100,
}
mock.register_uri(
'POST', requests_mock.ANY, json=response_location)
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
send_json = mock.request_history[0].json()
self.assertEqual(len(send_json['cellTowers']), 2)
self.assertEqual(len(send_json['wifiAccessPoints']), 3)
self.check_model_response(res, None, lat=1.0, lon=1.0, accuracy=100)
self.check_stats(
timer=[self.metric_url],
counter=[self.metric + '.api_key.test',
self.metric + '.fallback_hit',
self.metric_url + '.200',
self.metric + '.api_log.test.fallback_hit'],
)
def test_ok_no_existing_cell(self):
session = self.session
now_ms = int(time.time() * 1000)
first_of_month = utcnow().replace(day=1, hour=0, minute=0, second=0)
cell = CellFactory.build()
res = self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
cell.lat,
"longitude":
cell.lon,
"accuracy":
12.4,
"altitude":
100.1,
"altitudeAccuracy":
23.7,
"heading":
45.0,
"speed":
3.6,
"timestamp":
now_ms,
"cellTowers": [{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"age": 3,
"asu": 31,
"psc": cell.psc,
"signalStrength": -51,
"timingAdvance": 1,
}]
},
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {})
self.assertEquals(session.query(Cell).count(), 1)
result = session.query(CellObservation).all()
self.assertEquals(len(result), 1)
obs = result[0]
for name in ('lat', 'lon', 'radio', 'mcc', 'mnc', 'lac', 'cid', 'psc'):
self.assertEqual(getattr(obs, name), getattr(cell, name))
self.assertEqual(obs.accuracy, 12)
self.assertEqual(obs.altitude, 100)
self.assertEqual(obs.altitude_accuracy, 24)
self.assertEqual(obs.heading, 45.0)
self.assertEqual(obs.speed, 3.6)
self.assertEqual(obs.time, first_of_month)
self.assertEqual(obs.asu, 31)
self.assertEqual(obs.signal, -51)
self.assertEqual(obs.ta, 1)
def test_duplicated_cell_observations(self):
session = self.session
cell = CellFactory.build()
self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
cell.lat,
"longitude":
cell.lon,
"cellTowers": [
{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"asu": 10
},
{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"asu": 16
},
]
},
]
},
status=200)
self.assertEquals(session.query(CellObservation).count(), 1)
def test_single_cell_results_cached_preventing_external_call(self):
cell = CellFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_location)
query = self.model_query(cells=[cell])
query.cell[0]['signal'] = -77
location = self.provider.locate(query)
self.check_model_location(location, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(
counter=[
'm.fallback.lookup_status.200',
'm.fallback.cache.miss',
],
timer=['m.fallback.lookup'])
# vary the signal strength, not part of cache key
query.cell[0]['signal'] = -82
location = self.provider.locate(query)
self.check_model_location(location, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(
counter=[
'm.fallback.lookup_status.200',
'm.fallback.cache.hit',
],
timer=['m.fallback.lookup'])
def test_cell_geoip_mismatch(self):
# UK GeoIP with US mcc
cell = CellFactory.create(mcc=310)
query = self.model_query(cells=[cell])
res = self._call(body=query, ip=self.test_ip)
self.check_model_response(res, cell, country='US')
self.check_db_calls(rw=0, ro=0)
def test_no_call_made_when_not_allowed_for_apikey(self):
cells = CellFactory.build_batch(2)
wifis = WifiFactory.build_batch(2)
self.provider.api_key.allow_fallback = False
query = self.model_query(cells=cells, wifis=wifis)
self.check_should_locate(query, False)
def test_cache_single_cell(self):
cell = CellFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(cells=[cell])
query.cell[0].signal = -77
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
# vary the signal strength, not part of cache key
query.cell[0].signal = -82
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
def test_cache_empty_result(self):
cell = CellFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST',
requests_mock.ANY,
json=LocationNotFound.json_body(),
status_code=404
)
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup', ['status:404']),
])
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup', ['status:404']),
])
def test_set_cache_redis_failure(self):
cell = CellFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(self.source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertTrue(mock_redis_client.mget.called)
self.assertTrue(mock_redis_client.mset.called)
self.assertTrue(mock_request.called)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
])
def test_api_key_disallows(self):
api_key = ApiKeyFactory.build(allow_fallback=False)
cells = CellFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(query, False)
def test_success(self):
cell = CellFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
request_json = mock_request.request_history[0].json()
self.assertEqual(request_json['fallbacks'], {'lacf': True})
self.check_stats(counter=[
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
def test_database_error(self, db_errors=0):
for tablename in ('cell', 'cell_area',
'ocid_cell', 'ocid_cell_area'):
self.session.execute(text('drop table %s;' % tablename))
for i in range(16):
self.session.execute(text(
'drop table wifi_shard_%s;' % hex(i)[2:]))
cells = CellFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
self.check_stats(counter=[
(self.metric_type + '.result',
['key:test', 'country:GB',
'accuracy:high', 'status:miss']),
])
self.check_raven([('ProgrammingError', db_errors)])
def test_duplicated_cell_observations(self):
session = self.session
cell = CellFactory.build()
self.app.post_json(
'/v1/geosubmit?key=test',
{"items": [
{"latitude": cell.lat,
"longitude": cell.lon,
"cellTowers": [
{"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"asu": 10},
{"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"asu": 16},
]},
]},
status=200)
self.assertEquals(session.query(CellObservation).count(), 1)
def test_cell(self):
cell = CellFactory.build()
cell_query = {
'radio': cell.radio.name,
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
'psc': cell.psc,
'signal': -90,
'ta': 1,
}
query = Query(cell=[cell_query])
self.assertEqual(len(query.cell), 1)
query_cell = query.cell[0]
for key, value in cell_query.items():
if key == 'radio':
self.assertEqual(query_cell[key], cell.radio)
else:
self.assertEqual(query_cell[key], value)
self.assertEqual(len(query.cell_area), 1)
query_area = query.cell_area[0]
for key, value in cell_query.items():
if key == 'radio':
self.assertEqual(query_area[key], cell.radio)
elif key in ('cid', 'psc'):
pass
else:
self.assertEqual(query_area[key], value)
def test_cell_ambiguous(self):
# cell with ambiguous mcc to country mapping
cell = CellFactory.create(mcc=234)
query = self.model_query(cells=[cell])
res = self._call(body=query)
self.check_model_response(res, cell, country='GB')
self.check_db_calls(rw=0, ro=0)
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
cell = CellFactory.build()
self.session.add(
CellBlacklist(time=last_week, count=1,
radio=cell.radio, mcc=cell.mcc,
mnc=cell.mnc, lac=cell.lac, cid=cell.cid))
self.session.flush()
# add a new entry for the previously blacklisted cell
obs = dict(lat=cell.lat, lon=cell.lon,
radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid)
insert_measures_cell.delay([obs]).get()
self.assertEqual(self.data_queue.size(), 1)
update_cell.delay().get()
# the cell was inserted again
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(cells[0].created, last_week)
self.check_statcounter(StatKey.cell, 1)
self.check_statcounter(StatKey.unique_cell, 0)
def add_reports(self, number=3, api_key='test', email=None):
reports = []
for i in range(number):
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'cellTowers': [],
'wifiAccessPoints': [],
}
cell = CellFactory.build()
report['position']['latitude'] = cell.lat
report['position']['longitude'] = cell.lon
report['position']['accuracy'] = 17 + i
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiFactory.build_batch(2, lat=cell.lat, lon=cell.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi.key,
'signalStrength': -90 + i,
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
queue_reports.delay(reports=reports, api_key=api_key,
email=email).get()
return reports
def setUp(self):
super(TestFallbackProvider, self).setUp()
self.provider.api_key.allow_fallback = True
self.response_location = {
'location': {
'lat': 51.5366,
'lng': 0.03989,
},
'accuracy': 1500,
'fallback': 'lacf',
}
self.cells = []
for cell in CellFactory.build_batch(2):
self.cells.append({
'radio': cell.radio,
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
'signal': -70,
})
self.cells[0]['ta'] = 1
self.wifis = []
for wifi in WifiFactory.build_batch(2):
self.wifis.append({
'key': wifi.key,
'signal': -77,
})
self.wifis[0]['channel'] = 6
self.wifis[0]['frequency'] = 2437
self.wifis[0]['snr'] = 13
def test_blacklist(self):
now = util.utcnow()
cell = CellFactory.build()
observations = [dict(radio=int(cell.radio), mcc=cell.mcc,
mnc=cell.mnc, lac=cell.lac, cid=cell.cid + i,
psc=cell.psc,
lat=cell.lat + i * 0.0000001,
lon=cell.lon + i * 0.0000001)
for i in range(1, 4)]
black = CellBlacklist(
radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid + 1,
time=now, count=1,
)
self.session.add(black)
self.session.flush()
result = insert_measures_cell.delay(observations)
self.assertEqual(result.get(), 2)
self.assertEqual(self.data_queue.size(), 2)
update_cell.delay().get()
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.check_statcounter(StatKey.cell, 2)
self.check_statcounter(StatKey.unique_cell, 2)
def add_reports(self, number=3, api_key='test', email=None):
reports = []
for i in range(number):
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'cellTowers': [],
'wifiAccessPoints': [],
}
cell = CellFactory.build()
report['position']['latitude'] = cell.lat
report['position']['longitude'] = cell.lon
report['position']['accuracy'] = 17 + i
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiFactory.build_batch(2, lat=cell.lat, lon=cell.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi.key,
'signalStrength': -90 + i,
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
queue_reports.delay(
reports=reports, api_key=api_key, email=email).get()
return reports
def test_get_cell_multi(self):
cells = CellFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_unique_cell_histogram(self):
session = self.session
today = util.utcnow()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(3))
CellFactory(created=long_ago, radio=Radio.gsm)
CellFactory(created=two_days, radio=Radio.umts)
CellFactory(created=two_days, radio=Radio.umts, cid=50)
CellFactory(created=yesterday, radio=Radio.lte, cid=50)
CellFactory(created=today, radio=Radio.gsm, mnc=30)
session.flush()
result = unique_cell_histogram.delay(ago=3)
self.assertEqual(result.get(), 1)
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, StatKey.unique_cell)
self.assertEqual(stats[0].time, long_ago.date())
self.assertEqual(stats[0].value, 1)
# fill up newer dates
unique_cell_histogram.delay(ago=2).get()
unique_cell_histogram.delay(ago=1).get()
unique_cell_histogram.delay(ago=0).get()
# test duplicate execution
unique_cell_histogram.delay(ago=1).get()
stats = session.query(Stat.time, Stat.value).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(dict(stats), {
long_ago.date(): 1,
two_days.date(): 3,
yesterday.date(): 4,
today.date(): 5})
def test_database_error(self):
london = self.geoip_data['London']
session = self.session
stmt = text("drop table wifi;")
session.execute(stmt)
stmt = text("drop table cell;")
session.execute(stmt)
cell = CellFactory.build()
wifis = WifiFactory.build_batch(2)
res = self.app.post_json(
'/v1/geolocate?key=test', {
"cellTowers": [
{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
],
"wifiAccessPoints": [
{
"macAddress": wifis[0].key
},
{
"macAddress": wifis[1].key
},
]
},
extra_environ={'HTTP_X_FORWARDED_FOR': london['ip']},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(
res.json, {
"location": {
"lat": london['latitude'],
"lng": london['longitude']
},
"accuracy": london['accuracy']
})
self.check_stats(timer=['request.v1.geolocate'],
counter=[
'request.v1.geolocate.200',
'geolocate.geoip_hit',
])
self.check_raven([('ProgrammingError', 2)])
def test_missing_radio(self):
cell = CellFactory.build()
self.app.post_json('/v1/submit', {
'items': [{
'lat':
cell.lat,
'lon':
cell.lon,
'cell': [{
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
}]
}]
},
status=204)
self.assertEqual(self.session.query(CellObservation).count(), 0)
def test_no_api_key(self):
cell = CellFactory()
self.session.flush()
res = self.app.post_json(self.url, {
"cellTowers": [
{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid
},
]
},
status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(u'Invalid API key', res.json['error']['message'])
self.check_stats(counter=[self.metric + '.no_api_key'])
def test_invalid_cell(self):
cell = CellFactory.build()
self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
cell.lat,
"longitude":
cell.lon,
"cellTowers": [{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": 2000,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
}]
},
]
},
status=200)
self.assertEquals(self.session.query(CellObservation).count(), 0)
def test_ok_partial_cell(self):
session = self.session
cell = CellFactory()
session.flush()
res = self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
cell.lat,
"longitude":
cell.lon,
"cellTowers": [{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"psc": cell.psc
}, {
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"psc": cell.psc + 1,
}]
},
]
},
status=200)
# check that we get an empty response
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {})
observations = session.query(CellObservation).all()
self.assertEqual(len(observations), 2)
pscs = set([obs.psc for obs in observations])
self.assertEqual(pscs, set([cell.psc, cell.psc + 1]))
def test_ok_radioless_cell(self):
cell = CellFactory.build()
self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
cell.lat,
"longitude":
cell.lon,
"cellTowers": [{
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"psc": cell.psc,
}]
},
]
},
status=200)
observations = self.session.query(CellObservation).all()
self.assertEqual(len(observations), 0)
def test_ok_cell(self):
session = self.session
cell = CellFactory()
new_cell = CellFactory.build()
session.flush()
res = self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
cell.lat,
"longitude":
cell.lon,
"radioType":
cell.radio.name,
"cellTowers": [{
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
}]
},
{
"latitude":
new_cell.lat,
"longitude":
new_cell.lon,
"cellTowers": [{
"radioType": new_cell.radio.name,
"mobileCountryCode": new_cell.mcc,
"mobileNetworkCode": new_cell.mnc,
"locationAreaCode": new_cell.lac,
"cellId": new_cell.cid,
}]
},
]
},
status=200)
# check that we get an empty response
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {})
self.assertEqual(session.query(Cell).count(), 2)
observations = session.query(CellObservation).all()
self.assertEqual(len(observations), 2)
radios = set([obs.radio for obs in observations])
self.assertEqual(radios, set([cell.radio, new_cell.radio]))
self.check_stats(counter=[
'geosubmit.api_key.test',
'items.api_log.test.uploaded.batches',
'items.api_log.test.uploaded.reports',
'items.api_log.test.uploaded.cell_observations',
'items.uploaded.cell_observations',
'items.uploaded.batches',
'items.uploaded.reports',
'request.v1.geosubmit.200',
],
timer=[
'items.api_log.test.uploaded.batch_size',
'items.uploaded.batch_size',
'request.v1.geosubmit'
])
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(hours=1)
schema = ValidCellKeySchema()
obs_factory = CellObservationFactory
invalid_key = dict(lac=schema.fields['lac'].missing,
cid=schema.fields['cid'].missing)
cell1 = CellFactory(new_measures=3, total_measures=5)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory.create_batch(2, created=now, **invalid_key)
cell2 = CellFactory(lat=lat1 + 1.0,
lon=lon1 + 1.0,
new_measures=2,
total_measures=4)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(lac=cell2.lac, cid=cell2.cid)
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old observations are skipped
obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2)
obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellFactory(new_measures=10, total_measures=100000)
lat3, lon3 = (cell3.lat, cell3.lon)
obs_factory.create_batch(10,
lat=lat3 + 1.0,
lon=lon3 + 1.0,
**dict(lac=cell3.lac, cid=cell3.cid))
self.session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (3, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 3)
self.assertEqual(set([c.new_measures for c in cells]), set([0]))
for cell in cells:
if cell.hashkey() == cell1.hashkey():
self.assertEqual(cell.lat, lat1 + 0.002)
self.assertEqual(cell.lon, lon1 + 0.003)
if cell.hashkey() == cell2.hashkey():
self.assertEqual(cell.lat, lat2 + 0.001)
self.assertEqual(cell.lon, lon2 + 0.002)
if cell.hashkey() == cell3.hashkey():
expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010
expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010
self.assertAlmostEqual(cell.lat, expected_lat, 7)
self.assertAlmostEqual(cell.lon, expected_lon, 7)
