def test_database_error(self, db_errors=0):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, CellOCID, CellAreaOCID):
self.session.execute(text("drop table %s;" % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
self.session.execute(text("drop table %s;" % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text("drop table %s;" % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, "ok")
self.check_stats(
counter=[("request", [self.metric_path, "method:post", "status:200"])],
timer=[("request", [self.metric_path, "method:post"])],
)
if self.apikey_metrics:
self.check_stats(
counter=[
(
self.metric_type + ".result",
["key:test", "region:GB", "fallback_allowed:false", "accuracy:high", "status:miss"],
)
]
)
self.check_raven([("ProgrammingError", db_errors)])
def test_database_error(self, app, data_queues,
raven, session, stats, restore_db):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, ):
session.execute(text('drop table %s;' % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
session.execute(text('drop table %s;' % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
session.execute(text('drop table %s;' % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(app, body=query, ip=self.test_ip)
self.check_response(data_queues, res, 'ok', fallback='ipf')
self.check_queue(data_queues, 0)
stats.check(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
stats.check(counter=[
(self.metric_type + '.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
raven.check([('ProgrammingError', 3)])
def test_database_error(self, db_errors=0):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, CellOCID, CellAreaOCID):
self.session.execute(text('drop table %s;' % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
self.session.execute(text('drop table %s;' % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text('drop table %s;' % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
self.check_stats(counter=[
(self.metric_type + '.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
self.check_raven([('ProgrammingError', db_errors)])
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[{
'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'asu': 17,
'primaryScramblingCode': 5,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'
}])
data = self._call(query.json())
assert (data == {
'cells': [{
'radio': cell.radio.name,
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
'asu': 17,
'psc': 5,
'signal': -70,
'tA': 15,
}],
'fallbacks': {
'lacf': True
},
'token':
None,
})
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[{
'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'
}])
data = self._call(query.json())
assert (data == {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
}],
'considerIp':
False,
})
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
cell_query = self.cell_model_query([cell])
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 1)
self.assertEqual(query.expected_accuracy, DataAccuracy.medium)
query_cell = query.cell[0]
for key, value in cell_query[0].items():
query_value = getattr(query_cell, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
else:
self.assertEqual(query_value, value)
self.assertEqual(len(query.cell_area), 1)
query_area = query.cell_area[0]
for key, value in cell_query[0].items():
query_value = getattr(query_area, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
elif key in ('cid', 'psc'):
pass
else:
self.assertEqual(query_value, value)
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[
{'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'}])
data = self._call(query.json())
assert (data == {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
}],
'considerIp': False,
})
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[{
'radio': cell.radio,
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
'age': 1200,
'psc': 5,
'signal': -70,
'ta': 15,
'unknown_field': 'foo'
}])
data = self.schema.deserialize(query.internal_query())
self.assertEqual(
data, {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': 5,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
}],
'fallbacks': {
'lacf': True
},
})
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(
cell=[
{
"radioType": cell.radio,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"age": 1200,
"signalStrength": -70,
"timingAdvance": 15,
"unknown_field": "foo",
}
]
)
data = self._call(query.json())
assert data == {
"cellTowers": [
{
"radioType": cell.radio.name,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"age": 1200,
"signalStrength": -70,
"timingAdvance": 15,
}
],
"considerIp": False,
}
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[{
"radioType": cell.radio,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"asu": 17,
"primaryScramblingCode": 5,
"signalStrength": -70,
"timingAdvance": 15,
"unknown_field": "foo",
}])
data = self._call(query.json())
assert data == {
"cells": [{
"radio": cell.radio.name,
"mcc": cell.mcc,
"mnc": cell.mnc,
"lac": cell.lac,
"cid": cell.cid,
"asu": 17,
"psc": 5,
"signal": -70,
"tA": 15,
}],
"fallbacks": {
"lacf": True
},
"token":
None,
}
def test_dont_recache(self, geoip_db, http_session, session, source,
stats):
cell = CellShardFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = [self.fallback_cached_result]
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
with mock.patch.object(source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(geoip_db,
http_session,
session,
stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert not mock_redis_client.mset.called
stats.check(counter=[
('locate.fallback.cache', ['status:hit']),
])
def test_cache_single_cell(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(cells=[cell])
query.cell[0].signal = -77
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
# vary the signal strength, not part of cache key
query.cell[0].signal = -82
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
def test_set_cache_redis_failure(self):
cell = CellShardFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(self.source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertTrue(mock_redis_client.mget.called)
self.assertTrue(mock_redis_client.mset.called)
self.assertTrue(mock_request.called)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
])
def test_success(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
request_json = mock_request.request_history[0].json()
self.assertEqual(request_json['fallbacks'], {'lacf': True})
self.check_stats(counter=[
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[{
'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'asu': None,
'primaryScramblingCode': 5,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'
}])
data = OUTBOUND_SCHEMA.deserialize(query.json())
assert (data == {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': 5,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
}],
'fallbacks': {
'lacf': True
},
})
def test_cell_not_found(self, app, data_queues, stats):
cell = CellShardFactory.build()
query = self.model_query(cells=[cell])
res = self._call(app, body=query, status=self.not_found.code)
self.check_response(data_queues, res, 'not_found')
stats.check(counter=[
('request', [
self.metric_path, 'method:post',
'status:%s' % self.not_found.code
]),
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.query', [
'key:test', 'region:none', 'geoip:false', 'blue:none',
'cell:one', 'wifi:none'
]),
(self.metric_type + '.result', [
'key:test', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:miss'
]),
(self.metric_type + '.source', [
'key:test', 'region:none', 'source:internal',
'accuracy:medium', 'status:miss'
]),
],
timer=[
('request', [self.metric_path, 'method:post']),
])
def test_set_cache_redis_failure(
self, geoip_db, http_session, raven, session, source, metricsmock
):
cell = CellShardFactory.build()
mock_redis_client = _mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
"POST", requests_mock.ANY, json=self.fallback_result
)
with mock.patch.object(
source.caches[DEFAULT_SCHEMA], "redis_client", mock_redis_client
):
query = self.model_query(geoip_db, http_session, session, cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert mock_redis_client.mset.called
assert mock_request.called
raven.check([("RedisError", 1)])
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:miss"],
)
def test_cell_not_found(self, app, data_queues, metricsmock, logs):
"""A failed cell-based lookup emits several metrics."""
cell = CellShardFactory.build()
query = self.model_query(cells=[cell])
res = self._call(app, body=query, status=404)
self.check_response(data_queues, res, "not_found")
metricsmock.assert_incr_once(
"request", tags=[self.metric_path, "method:post", "status:404"]
)
metricsmock.assert_incr_once(
self.metric_type + ".request", tags=[self.metric_path, "key:test"]
)
metricsmock.assert_incr_once(
self.metric_type + ".query",
tags=["key:test", "geoip:false", "blue:none", "cell:one", "wifi:none"],
)
metricsmock.assert_incr_once(
self.metric_type + ".result",
tags=[
"key:test",
"fallback_allowed:false",
"accuracy:medium",
"status:miss",
],
)
metricsmock.assert_incr_once(
self.metric_type + ".source",
tags=["key:test", "source:internal", "accuracy:medium", "status:miss"],
)
assert logs.only_entry["cell_valid"] == 1
def test_cell(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell])
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 1)
self.assertEqual(query.expected_accuracy, DataAccuracy.medium)
query_cell = query.cell[0]
for key, value in cell_query[0].items():
query_value = getattr(query_cell, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
else:
self.assertEqual(query_value, value)
self.assertEqual(len(query.cell_area), 1)
query_area = query.cell_area[0]
for key, value in cell_query[0].items():
query_value = getattr(query_area, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
elif key in ('cid', 'psc'):
pass
else:
self.assertEqual(query_value, value)
def test_dont_recache(self, geoip_db, http_session,
session, source, stats):
cell = CellShardFactory.build()
mock_redis_client = _mock_redis_client()
mock_redis_client.mget.return_value = [self.fallback_cached_result]
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(source.caches[DEFAULT_SCHEMA],
'redis_client',
mock_redis_client):
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert not mock_redis_client.mset.called
stats.check(counter=[
('locate.fallback.cache', [self.fallback_tag, 'status:hit']),
])
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[
{'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'asu': 17,
'primaryScramblingCode': 5,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'}])
data = self._call(query.json())
assert (data == {
'cells': [{
'radio': cell.radio.name,
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
'asu': 17,
'psc': 5,
'signal': -70,
'tA': 15,
}],
'fallbacks': {'lacf': True},
'token': None,
})
def test_set_cache_redis_failure(self, geoip_db, http_session,
raven, session, source, stats):
cell = CellShardFactory.build()
mock_redis_client = _mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(source.caches[DEFAULT_SCHEMA],
'redis_client',
mock_redis_client):
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert mock_redis_client.mset.called
assert mock_request.called
raven.check([('RedisError', 1)])
stats.check(counter=[
('locate.fallback.cache', [self.fallback_tag, 'status:miss']),
])
def test_success(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
query = self.model_query(
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = self.source.search(query)
self.check_model_results(results, [self.fallback_model])
self.assertAlmostEqual(results.best().score, 5.0, 4)
request_json = mock_request.request_history[0].json()
self.assertEqual(request_json['fallbacks'], {'lacf': True})
self.check_stats(counter=[
('locate.fallback.lookup', ['fallback_name:fall', 'status:200']),
],
timer=[
('locate.fallback.lookup', ['fallback_name:fall'
]),
])
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key,
'nickname': nickname,
'report': rep} for rep in reports]
self.incoming_queue.enqueue(items)
update_incoming.delay().get()
return reports
def test_success(self, geoip_db, http_session, session, source, metricsmock):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
"POST", requests_mock.ANY, json=self.fallback_result
)
query = self.model_query(
geoip_db,
http_session,
session,
cells=[cell],
fallback={"lacf": True, "ipf": False},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
self._check_success_fallbacks(request_json)
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:miss"],
)
assert metricsmock.has_record(
"timing", "locate.fallback.lookup.timing", tags=[self.fallback_tag]
)
def test_dont_recache(self, geoip_db, http_session, session, source, metricsmock):
cell = CellShardFactory.build()
mock_redis_client = _mock_redis_client()
mock_redis_client.mget.return_value = [self.fallback_cached_result]
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
"POST", requests_mock.ANY, json=self.fallback_result
)
with mock.patch.object(
source.caches[DEFAULT_SCHEMA], "redis_client", mock_redis_client
):
query = self.model_query(geoip_db, http_session, session, cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert not mock_redis_client.mset.called
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:hit"],
)
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
cell_query = self.cell_model_query([cell])
query = Query(cell=cell_query)
assert len(query.cell) == 1
assert query.expected_accuracy is DataAccuracy.medium
assert query.geoip_only is False
query_cell = query.cell[0]
for key, value in cell_query[0].items():
query_value = getattr(query_cell, key, None)
if key == 'radioType':
assert query_value is cell.radio
else:
assert query_value == value
assert len(query.cell_area) == 1
query_area = query.cell_area[0]
for key, value in cell_query[0].items():
query_value = getattr(query_area, key, None)
if key == 'radioType':
assert query_value is cell.radio
elif key in ('cellId', 'primaryScramblingCode'):
pass
else:
assert query_value == value
def test_success(self, geoip_db, http_session, session, source, stats):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
self._check_success_fallbacks(request_json)
stats.check(counter=[
('locate.fallback.lookup', [self.fallback_tag, 'status:200']),
], timer=[
('locate.fallback.lookup', [self.fallback_tag]),
])
def test_cache_empty_result(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST',
requests_mock.ANY,
json=LocationNotFound.json_body(),
status_code=404
)
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup', ['status:404']),
])
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup', ['status:404']),
])
def test_set_cache_redis_failure(self, geoip_db, http_session, raven,
session, source, stats):
cell = CellShardFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
with mock.patch.object(source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(geoip_db,
http_session,
session,
stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert mock_redis_client.mset.called
assert mock_request.called
raven.check([('RedisError', 1)])
stats.check(counter=[
('locate.fallback.cache', ['status:miss']),
])
def test_success(self, geoip_db, http_session, session, source, stats):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
query = self.model_query(
geoip_db,
http_session,
session,
stats,
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
assert request_json['fallbacks'] == {'lacf': True}
stats.check(counter=[
('locate.fallback.lookup', ['fallback_name:fall', 'status:200']),
],
timer=[
('locate.fallback.lookup', ['fallback_name:fall']),
])
def test_cache_empty_result(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=LocationNotFound.json_body(),
status_code=404)
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup',
['fallback_name:fall', 'status:404']),
])
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup',
['fallback_name:fall', 'status:404']),
])
def test_cache_single_cell(
self, geoip_db, http_session, session, source, metricsmock
):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
"POST", requests_mock.ANY, json=self.fallback_result
)
query = self.model_query(geoip_db, http_session, session, cells=[cell])
query.cell[0].signalStrength = -77
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
assert mock_request.call_count == 1
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=["fallback_name:fall", "status:miss"],
)
assert metricsmock.has_record(
"incr",
"locate.fallback.lookup",
value=1,
tags=["fallback_name:fall", "status:200"],
)
assert metricsmock.has_record(
"timing", "locate.fallback.lookup.timing", tags=["fallback_name:fall"]
)
metricsmock.clear_records()
# vary the signal strength, not part of cache key
query.cell[0].signalStrength = -82
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
assert mock_request.call_count == 1
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=["fallback_name:fall", "status:hit"],
)
assert metricsmock.has_record(
"incr",
"locate.source",
value=1,
tags=[
"key:test",
"region:none",
"source:fallback",
"accuracy:medium",
"status:hit",
],
)
def test_local_export(self, celery, session):
now = util.utcnow()
today = now.date()
long_ago = now - timedelta(days=367)
cell_fixture_fields = ("radio", "cid", "lat", "lon", "mnc", "mcc", "lac")
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {
"radio": Radio.wcdma,
"mcc": base_cell.mcc,
"mnc": base_cell.mnc,
"lac": base_cell.lac,
}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=base_cell.lat, lon=base_cell.lon, **cell_key)
CellShardFactory(**cell)
cell["lat"] = "%.7f" % cell["lat"]
cell["lon"] = "%.7f" % cell["lon"]
cell["radio"] = "UMTS"
cell_strings = [(field, str(value)) for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
# add one really old cell
CellShardFactory(
cid=220,
created=long_ago,
modified=long_ago,
last_seen=long_ago.date(),
**cell_key,
)
session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, "export.csv.gz")
write_stations_to_csv(session, path, today)
with util.gzip_open(path, "r") as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = next(reader)
assert "area" in header.values()
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
assert cells == exported_cells
def test_cell_area_duplicated(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell, cell, cell])
cell_query[1]['cid'] += 2
cell_query[2]['cid'] += 1
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 3)
self.assertEqual(len(query.cell_area), 1)
def test_check_invalid_cell(self, geoip_db, http_session, session, source):
malformed_cell = CellShardFactory.build()
malformed_cell.mcc = 99999
query = self.model_query(
geoip_db, http_session, session, cells=[malformed_cell]
)
self.check_should_search(source, query, False)
def test_cell_area_duplicated(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell, cell, cell])
cell_query[1]['cid'] += 2
cell_query[2]['cid'] += 1
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 3)
self.assertEqual(len(query.cell_area), 1)
def _one_cell_query(self, radio=True):
cell = CellShardFactory.build()
query = {'lat': cell.lat, 'lon': cell.lon,
'cell': [{'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid}]}
if radio:
query['cell'][0]['radio'] = cell.radio.name
return (cell, query)
def test_cell_area_duplicated(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell, cell, cell])
cell_query[1]['cellId'] += 2
cell_query[2]['cellId'] += 1
query = Query(cell=cell_query)
assert len(query.cell) == 3
assert len(query.cell_area) == 1
def _one_cell_query(self, radio=True):
cell = CellShardFactory.build()
query = {'lat': cell.lat, 'lon': cell.lon,
'cell': [{'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid}]}
if radio:
query['cell'][0]['radio'] = cell.radio.name
return (cell, query)
def test_cell_duplicated(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell, cell, cell])
cell_query[0]['signalStrength'] = -95
cell_query[1]['signalStrength'] = -90
cell_query[2]['signalStrength'] = -92
query = Query(cell=cell_query)
assert len(query.cell) == 1
assert query.cell[0].signalStrength == -90
def test_no_api_key_shortname(self):
api_key = ApiKeyFactory.build(shortname=None, log_locate=True)
cell = CellShardFactory.build()
self._make_query(api_key=api_key, cell=[cell])
self.check_stats(counter=[
('locate.query',
['key:%s' % api_key.valid_key,
'region:none', 'geoip:false', 'cell:one', 'wifi:none']),
])
def test_cell_radio(self):
cell = CellShardFactory.build()
data = SUBMIT_V0_SCHEMA.deserialize({'items': [
{'lat': cell.lat, 'lon': cell.lon, 'cell': [{
'radio': 'UMTS', 'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid,
}]}
]})
assert data['items'][0]['cellTowers'][0]['radioType'] == 'wcdma'
cell = CellShardFactory.build()
data = SUBMIT_V0_SCHEMA.deserialize({'items': [
{'lat': cell.lat, 'lon': cell.lon, 'cell': [{
'radio': 'foo', 'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid,
}]}
]})
assert 'radioType' not in data['items'][0]['cellTowers'][0]
def test_cell_geoip_mismatch(self, app, session):
# UK GeoIP with ambiguous US mcc
uk_cell = CellShardFactory.build(mcc=234)
us_cell = CellShardFactory(mcc=310)
session.flush()
query = self.model_query(cells=[us_cell])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, uk_cell, region="GB", fallback="ipf")
def test_cell_duplicated(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell, cell, cell])
cell_query[0]['signal'] = -95
cell_query[1]['signal'] = -90
cell_query[2]['signal'] = -92
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 1)
self.assertEqual(query.cell[0].signal, -90)
def test_no_api_key_shortname(self):
api_key = ApiKeyFactory.build(shortname=None, log_locate=True)
cell = CellShardFactory.build()
self._make_query(api_key=api_key, cell=[cell])
self.check_stats(counter=[
('locate.query',
['key:%s' % api_key.valid_key,
'region:none', 'geoip:false', 'cell:one', 'wifi:none']),
])
def test_cell_geoip_mismatch(self, app, session):
# UK GeoIP with ambiguous US mcc
uk_cell = CellShardFactory.build(mcc=234)
us_cell = CellShardFactory(mcc=310)
session.flush()
query = self.model_query(cells=[us_cell])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, uk_cell, region='GB', fallback='ipf')
def test_cell_radio(self):
cell = CellShardFactory.build()
data = SUBMIT_V0_SCHEMA.deserialize({'items': [
{'lat': cell.lat, 'lon': cell.lon, 'cell': [{
'radio': 'UMTS', 'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid,
}]}
]})
assert data['items'][0]['cellTowers'][0]['radioType'] == 'wcdma'
cell = CellShardFactory.build()
data = SUBMIT_V0_SCHEMA.deserialize({'items': [
{'lat': cell.lat, 'lon': cell.lon, 'cell': [{
'radio': 'foo', 'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid,
}]}
]})
assert 'radioType' not in data['items'][0]['cellTowers'][0]
def test_cell_invalid_lac(self, app, data_queues, logs):
"""A valid CID with and invalid LAC is not an error."""
cell = CellShardFactory.build(radio=Radio.wcdma, lac=0, cid=1)
query = self.model_query(cells=[cell])
res = self._call(app, body=query, status=404)
self.check_response(data_queues, res, "not_found")
assert logs.only_entry["cell"] == 1
assert logs.only_entry["cell_valid"] == 0
def test_check_invalid_cell(self, geoip_db, http_session,
session, source, stats):
malformed_cell = CellShardFactory.build()
malformed_cell.mcc = 99999
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[malformed_cell])
self.check_should_search(source, query, False)
def test_invalid_json(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=['invalid json'])
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, None)
def test_malformed_json(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, content=b'[invalid json')
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
def test_ambiguous_mcc(self):
regions = GEOCODER.regions_for_mcc(234, metadata=True)
cell = CellShardFactory.build(mcc=234)
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_result(results, regions)
self.check_stats(counter=[
(self.api_type + '.source',
['key:test', 'region:none', 'source:internal',
'accuracy:low', 'status:hit']),
])
def test_import_local_ocid(self, celery, redis, session):
self.import_csv(
celery, redis, session,
CellShardFactory.build(radio=Radio.wcdma))
cells = session.query(CellShardOCID.shards()['wcdma']).all()
assert len(cells) == 9
areaids = set([cell.areaid for cell in cells])
assert session.query(CellAreaOCID).count() == len(areaids)
update_statcounter.delay().get()
self.check_stat(session, StatKey.unique_cell_ocid, 9)
def test_set_cell_not_found(self):
cell = CellShardFactory.build()
query = Query(cell=self.cell_model_query([cell]))
result = ExternalResult(None, None, None, None)
self.cache.set(query, result)
keys = self.redis_client.keys('cache:fallback:cell:*')
self.assertEqual(len(keys), 1)
self.assertEqual(self.redis_client.get(keys[0]), b'"404"')
self.assertEqual(self.cache.get(query), result)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:hit']),
])
def test_cell(self, app, celery):
now = util.utcnow()
today = now.replace(hour=0, minute=0, second=0)
cell = CellShardFactory.build(radio=Radio.umts)
res = self._post(app, [{
'lat': cell.lat,
'lon': cell.lon,
'time': today.strftime('%Y-%m-%d'),
'accuracy': 10.6,
'altitude': 123.1,
'altitude_accuracy': 7.0,
'heading': 45.2,
'pressure': 1020.23,
'speed': 3.6,
'source': 'gnss',
'radio': cell.radio.name,
'cell': [{
'radio': 'umts', 'mcc': cell.mcc,
'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid,
'age': 1000, 'asu': 3, 'psc': 7, 'serving': 1,
'signal': -85, 'ta': 2}],
}], api_key='test')
assert res.body == b''
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item['api_key'] == 'test'
report = item['report']
timestamp = datetime.utcfromtimestamp(report['timestamp'] / 1000.0)
timestamp = timestamp.replace(microsecond=0, tzinfo=pytz.UTC)
assert timestamp == today
position = report['position']
assert position['latitude'] == cell.lat
assert position['longitude'] == cell.lon
assert position['accuracy'] == 10.6
assert position['altitude'] == 123.1
assert position['altitudeAccuracy'] == 7.0
assert position['heading'] == 45.2
assert position['pressure'] == 1020.23
assert position['speed'] == 3.6
assert position['source'] == 'gnss'
cells = report['cellTowers']
assert cells[0]['radioType'] == 'wcdma'
assert cells[0]['mobileCountryCode'] == cell.mcc
assert cells[0]['mobileNetworkCode'] == cell.mnc
assert cells[0]['locationAreaCode'] == cell.lac
assert cells[0]['cellId'] == cell.cid
assert cells[0]['age'] == 1000
assert cells[0]['asu'] == 3
assert cells[0]['primaryScramblingCode'] == 7
assert cells[0]['serving'] == 1
assert cells[0]['signalStrength'] == -85
assert cells[0]['timingAdvance'] == 2
def test_set_cell(self):
cell = CellShardFactory.build()
query = Query(cell=self.cell_model_query([cell]))
result = ExternalResult(cell.lat, cell.lon, cell.radius, None)
self.cache.set(query, result)
keys = self.redis_client.keys('cache:fallback:cell:*')
self.assertEqual(len(keys), 1)
self.assertTrue(500 < self.redis_client.ttl(keys[0]) <= 600)
self.assertEqual(self.cache.get(query), result)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:hit']),
])