def test_get_cell_multi(self):
cells = CellShardFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_get_cell_multi(self, cache, metricsmock):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
assert cache.get(query) is None
metricsmock.assert_incr_once(
"locate.fallback.cache",
tags=[self.fallback_tag, "status:bypassed"])
def test_api_key_disallows(self):
api_key = ApiKeyFactory.build(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(query, False)
def test_get_cell_multi(self):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_get_cell_multi(self, cache, stats):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_mixed_cell_wifi(self):
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
self.assertEqual(query.expected_accuracy, DataAccuracy.high)
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key,
'nickname': nickname,
'report': rep} for rep in reports]
self.incoming_queue.enqueue(items)
update_incoming.delay().get()
return reports
def test_api_key_disallows(self):
api_key = ApiKeyFactory.build(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(query, False)
def test_medium_hit(self):
cells = CellShardFactory.build_batch(1)
self._make_query(self._make_result(accuracy=50000.0), cell=cells)
self.check_stats(counter=[
('locate.result',
['key:key', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:hit']),
])
def test_get_cell_unwiredlabs(self, unwiredlabs_cache, metricsmock):
cells = CellShardFactory.build_batch(1)
query = self._query(api_key=UNWIREDLABS_KEY,
cell=self.cell_model_query(cells))
assert unwiredlabs_cache.get(query) is None
metricsmock.assert_incr_once(
"locate.fallback.cache",
tags=["fallback_name:labs", "status:miss"])
def test_medium_hit(self):
cells = CellShardFactory.build_batch(1)
self._make_query(self._make_result(accuracy=50000.0), cell=cells)
self.check_stats(counter=[
('locate.result',
['key:key', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:hit']),
])
def test_mixed_cell_wifi(self):
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(
cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
self.assertEqual(query.expected_accuracy, DataAccuracy.high)
def test_get_cell_multi(self, cache, stats):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1,
[self.fallback_tag, 'status:bypassed']),
])
def test_mixed_cell_wifi(self):
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
assert query.expected_accuracy is DataAccuracy.high
assert query.geoip_only is False
def test_many(self):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(3)
self._make_query(cell=cells, wifi=wifis, ip=self.london_ip)
self.check_stats(total=1, counter=[
('locate.query',
['key:key', 'region:GB', 'cell:many', 'wifi:many']),
])
def test_get_cell_unwiredlabs(self, unwiredlabs_cache, stats):
cells = CellShardFactory.build_batch(1)
query = self._query(api_key=UNWIREDLABS_KEY,
cell=self.cell_model_query(cells))
assert unwiredlabs_cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1,
['fallback_name:labs', 'status:miss']),
])
def test_medium_miss(self, geoip_db, stats):
cells = CellShardFactory.build_batch(1)
self._make_query(geoip_db, stats, self._make_result(), cell=cells)
stats.check(counter=[
('locate.result', [
'key:test', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:miss'
]),
])
def test_mixed_cell_wifi(self):
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(
cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
assert query.expected_accuracy is DataAccuracy.high
assert query.geoip_only is False
def test_api_key_disallows(self, geoip_db, http_session, session, source):
api_key = KeyFactory(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(
geoip_db, http_session, session, cells=cells, wifis=wifis, api_key=api_key
)
self.check_should_search(source, query, False)
def test_get_cell_unwiredlabs(self, unwiredlabs_cache, stats):
cells = CellShardFactory.build_batch(1)
query = self._query(api_key=UNWIREDLABS_KEY,
cell=self.cell_model_query(cells))
assert unwiredlabs_cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1,
['fallback_name:labs', 'status:miss']),
])
def test_many(self):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(3)
self._make_query(cell=cells, wifi=wifis, ip=self.london_ip)
self.check_stats(total=1, counter=[
('locate.query',
['key:key', 'region:GB', 'cell:many', 'wifi:many']),
])
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
blue_key=None, cell_mcc=None, wifi_key=None,
api_key='test', lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key, 'report': rep} for rep in reports]
self.queue.enqueue(items)
return reports
def test_api_key_disallows(self, geoip_db, http_session,
session, source, stats):
api_key = KeyFactory(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(
geoip_db, http_session, session, stats,
cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(source, query, False)
def test_medium_miss_low(self, geoip_db, stats):
cells = CellShardFactory.build_batch(1)
self._make_query(
geoip_db, stats,
self._make_result(accuracy=50000.1), cell=cells)
stats.check(counter=[
('locate.result',
['key:test', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:miss']),
])
def test_mixed_hit(self, geoip_db, stats):
cells = CellShardFactory.build_batch(2)
self._make_query(
geoip_db, stats,
self._make_result(accuracy=500.0), cell=cells, ip=self.london_ip)
stats.check(counter=[
('locate.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:medium', 'status:hit']),
])
def test_apikey_error(self, db_errors=0):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
self.session.execute(text('drop table %s;' % ApiKey.__tablename__))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_raven([('ProgrammingError', db_errors)])
def test_apikey_error(self, db_errors=0):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
self.session.execute(text('drop table %s;' % ApiKey.__tablename__))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_raven([('ProgrammingError', db_errors)])
def test_many(self, geoip_db, metricsmock):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(3)
self._make_query(
geoip_db, blue=blues, cell=cells, wifi=wifis, ip=self.london_ip
)
metricsmock.assert_incr_once(
"locate.query", tags=["key:test", "blue:many", "cell:many", "wifi:many"]
)
def test_one(self):
blues = BlueShardFactory.build_batch(1)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(1)
self._make_query(blue=blues, cell=cells, wifi=wifis, ip=self.london_ip)
self.check_stats(total=1, counter=[
('locate.query',
['key:key', 'region:GB',
'blue:one', 'cell:one', 'wifi:one']),
])
def test_apikey_error(self, app, data_queues, raven, session, restore_db):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
session.execute(text("drop table %s;" % ApiKey.__tablename__))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(app, body=query, ip=self.test_ip)
self.check_response(data_queues, res, "ok", fallback="ipf")
raven.check([("ProgrammingError", 1)])
self.check_queue(data_queues, 0)
def test_apikey_error(self, app, data_queues,
raven, session, stats, restore_db):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
session.execute(text('drop table %s;' % ApiKey.__tablename__))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(app, body=query, ip=self.test_ip)
self.check_response(data_queues, res, 'ok', fallback='ipf')
raven.check([('ProgrammingError', 1)])
self.check_queue(data_queues, 0)
def test_medium_hit(self, geoip_db, metricsmock):
cells = CellShardFactory.build_batch(1)
self._make_query(geoip_db, self._make_result(accuracy=50000.0), cell=cells)
metricsmock.assert_incr_once(
"locate.result",
tags=[
"key:test",
"fallback_allowed:false",
"accuracy:medium",
"status:hit",
],
)
def test_many(self, geoip_db, stats):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(3)
self._make_query(
geoip_db, stats,
blue=blues, cell=cells, wifi=wifis, ip=self.london_ip)
stats.check(total=1, counter=[
('locate.query',
['key:test', 'region:GB',
'blue:many', 'cell:many', 'wifi:many']),
])
def test_one(self):
blues = BlueShardFactory.build_batch(1)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(1)
self._make_query(blue=blues, cell=cells, wifi=wifis, ip=self.london_ip)
self.check_stats(total=1,
counter=[
('locate.query', [
'key:key', 'region:GB', 'blue:one',
'cell:one', 'wifi:one'
]),
])
def test_get_cell_unwiredlabs(self, unwiredlabs_cache, metricsmock):
cells = CellShardFactory.build_batch(1)
query = self._query(api_key=UNWIREDLABS_KEY, cell=self.cell_model_query(cells))
assert unwiredlabs_cache.get(query) is None
assert (
len(
metricsmock.filter_records(
"incr",
"locate.fallback.cache",
value=1,
tags=["fallback_name:labs", "status:miss"],
)
)
== 1
)
def test_medium_miss(self, geoip_db, metricsmock):
cells = CellShardFactory.build_batch(1)
self._make_query(geoip_db, self._make_result(), cell=cells)
assert metricsmock.has_record(
"incr",
"locate.result",
value=1,
tags=[
"key:test",
"region:none",
"fallback_allowed:false",
"accuracy:medium",
"status:miss",
],
)
def test_get_cell_multi(self, cache, metricsmock):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
assert cache.get(query) is None
assert (
len(
metricsmock.filter_records(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:bypassed"],
)
)
== 1
)
def test_many(self, geoip_db, metricsmock):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(3)
self._make_query(geoip_db,
blue=blues,
cell=cells,
wifi=wifis,
ip=self.london_ip)
assert metricsmock.get_records() == [(
"incr",
"locate.query",
1,
["key:test", "region:GB", "blue:many", "cell:many", "wifi:many"],
)]
def test_mixed_hit(self, geoip_db, metricsmock):
cells = CellShardFactory.build_batch(2)
self._make_query(geoip_db,
self._make_result(accuracy=500.0),
cell=cells,
ip=self.london_ip)
assert metricsmock.has_record(
"incr",
"locate.result",
value=1,
tags=[
"key:test",
"region:GB",
"fallback_allowed:false",
"accuracy:medium",
"status:hit",
],
)
def test_get_mixed(self):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
self.assertEqual(self.cache.get(query), None)
query = Query(blue=self.blue_model_query(blues),
cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
query = Query(blue=self.blue_model_query(blues),
wifi=self.wifi_model_query(wifis))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 3, 1, ['status:bypassed']),
])
def test_get_mixed(self, cache, stats):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
assert cache.get(query) is None
query = self._query(blue=self.blue_model_query(blues),
cell=self.cell_model_query(cells))
assert cache.get(query) is None
query = self._query(blue=self.blue_model_query(blues),
wifi=self.wifi_model_query(wifis))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 3, 1, ['status:bypassed']),
])
def test_get_mixed(self, cache, metricsmock, mix1, mix2):
"""A fallback query with mixed station types is not cached."""
kwargs = {}
mix = set((mix1, mix2))
assert len(mix) == 2
if "cell" in mix:
kwargs["cell"] = self.cell_model_query(
CellShardFactory.build_batch(1))
if "blue" in mix:
kwargs["blue"] = self.blue_model_query(
BlueShardFactory.build_batch(2))
if "wifi" in mix:
kwargs["wifi"] = self.wifi_model_query(
WifiShardFactory.build_batch(2))
query = self._query(**kwargs)
assert cache.get(query) is None
metricsmock.assert_incr_once(
"locate.fallback.cache",
tags=[self.fallback_tag, "status:bypassed"])
def test_get_mixed(self, cache, stats):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
assert cache.get(query) is None
query = self._query(blue=self.blue_model_query(blues),
cell=self.cell_model_query(cells))
assert cache.get(query) is None
query = self._query(blue=self.blue_model_query(blues),
wifi=self.wifi_model_query(wifis))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 3, 1,
[self.fallback_tag, 'status:bypassed']),
])
def test_get_mixed(self, cache, metricsmock):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = self._query(
cell=self.cell_model_query(cells), wifi=self.wifi_model_query(wifis)
)
assert cache.get(query) is None
query = self._query(
blue=self.blue_model_query(blues), cell=self.cell_model_query(cells)
)
assert cache.get(query) is None
query = self._query(
blue=self.blue_model_query(blues), wifi=self.wifi_model_query(wifis)
)
assert cache.get(query) is None
assert metricsmock.get_records() == [
(
"incr",
"locate.fallback.cache",
1,
[self.fallback_tag, "status:bypassed"],
),
(
"incr",
"locate.fallback.cache",
1,
[self.fallback_tag, "status:bypassed"],
),
(
"incr",
"locate.fallback.cache",
1,
[self.fallback_tag, "status:bypassed"],
),
]
def test_apikey_error(self, app, data_queues, raven, session, restore_db, logs):
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
session.execute(text("drop table %s;" % ApiKey.__tablename__))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(app, body=query, ip=self.test_ip)
self.check_response(data_queues, res, "ok", fallback="ipf")
raven.check([("ProgrammingError", 1)])
self.check_queue(data_queues, 0)
log = logs.only_entry
expected_entry = {
"api_key": "test",
"api_key_db_fail": True,
"api_path": "v1.geolocate",
"api_repeat_response": False,
"api_response_sig": log["api_response_sig"],
"api_type": "locate",
"blue": 0,
"blue_valid": 0,
"cell": 2,
"cell_valid": 2,
"duration_s": log["duration_s"],
"event": "POST /v1/geolocate - 200",
"has_geoip": True,
"has_ip": True,
"http_method": "POST",
"http_path": "/v1/geolocate",
"http_status": 200,
"log_level": "info",
"region": "GB",
"wifi": 2,
"wifi_valid": 2,
}
assert log == expected_entry
def add_reports(
self,
celery,
num=1,
blue_factor=0,
cell_factor=1,
wifi_factor=2,
blue_key=None,
cell_mcc=None,
wifi_key=None,
api_key="test",
lat=None,
lon=None,
source=None,
):
reports = []
timestamp = int(time.time() * 1000)
for i in range(num):
pos = CellShardFactory.build()
report = {
"timestamp": timestamp,
"position": {},
"bluetoothBeacons": [],
"cellTowers": [],
"wifiAccessPoints": [],
}
report["position"]["latitude"] = lat or pos.lat
report["position"]["longitude"] = lon or pos.lon
report["position"]["accuracy"] = 17.0 + i
if source is not None:
report["position"]["source"] = source
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat,
lon=pos.lon)
for blue in blues:
blue_data = {
"macAddress": blue_key or blue.mac,
"signalStrength": -100 + i,
}
report["bluetoothBeacons"].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat,
lon=pos.lon)
for cell in cells:
cell_data = {
"radioType": cell.radio.name,
"mobileCountryCode": cell_mcc or cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
"cellId": cell.cid,
"primaryScramblingCode": cell.psc,
"signalStrength": -110 + i,
}
report["cellTowers"].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat,
lon=pos.lon)
for wifi in wifis:
wifi_data = {
"macAddress": wifi_key or wifi.mac,
"signalStrength": -90 + i,
"ssid": "my-wifi",
}
report["wifiAccessPoints"].append(wifi_data)
reports.append(report)
items = [{
"api_key": api_key,
"source": rep["position"].get("source", "gnss"),
"report": rep,
} for rep in reports]
self.queue(celery).enqueue(items)
return reports