def test_blue_seen(self, app, data_queues, session, mac):
"""If a query contains no new data, it is not queued for further processing.
Due to an issue with truncating bytestrings in a numpy array (numpy
issue 8089), addresses that end in '00' were once detected as new
stations.
"""
self.check_queue(data_queues, 0)
blue = BlueShardFactory(mac=mac)
offset = 0.00002
blues = [blue, BlueShardFactory(lat=blue.lat + offset)]
session.flush()
self.check_queue(data_queues, 0)
query = self.model_query(blues=blues)
res = self._call(app, body=query)
self.check_model_response(res, blue, lat=blue.lat + offset / 2)
if data_queues["update_incoming"].size():
items = data_queues["update_incoming"].dequeue()
pytest.fail(
(
"Query added a report to the update_incoming queue."
"\nQuery:\n{}\nReport:\n{}"
).format(query, items)
)
def test_blue(self, geoip_db, http_session, session, source, metricsmock):
now = util.utcnow()
region = GEOCODER.regions_for_mcc(235, metadata=True)[0]
blue1 = BlueShardFactory(samples=10)
blue2 = BlueShardFactory(samples=20)
blue3 = BlueShardFactory.build(region="DE", samples=100)
session.flush()
query = self.model_query(
geoip_db, http_session, session, blues=[blue1, blue2, blue3]
)
results = source.search(query)
self.check_model_results(results, [region])
best_result = results.best()
assert best_result.region_code == region.code
assert best_result.score == station_score(blue1, now) + station_score(
blue2, now
)
assert metricsmock.has_record(
"incr",
self.api_type + ".source",
value=1,
tags=[
"key:test",
"region:none",
"source:internal",
"accuracy:low",
"status:hit",
],
)
def test_update(self, celery, session):
area = CellAreaFactory(radio=Radio.gsm, num_cells=1)
area.region = None
BlueShardFactory.create_batch(2, region='CA')
BlueShardFactory.create_batch(3, region='GB')
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1)
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2)
CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2)
CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3)
CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4)
WifiShardFactory.create_batch(5, region='DE')
WifiShardFactory.create_batch(6, region='US')
wifi = WifiShardFactory()
wifi.region = None
session.add(RegionStat(region='US', blue=1, wifi=2))
session.add(RegionStat(region='TW', wifi=1))
session.flush()
update_statregion.delay().get()
stats = session.query(RegionStat).all()
assert len(stats) == 4
for stat in stats:
values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi)
if stat.region == 'DE':
assert values == (3, 3, 0, 0, 5)
elif stat.region == 'CA':
assert values == (2, 0, 4, 2, 0)
elif stat.region == 'GB':
assert values == (0, 0, 0, 3, 0)
elif stat.region == 'US':
assert values == (0, 0, 0, 0, 6)
def test_update(self):
area = CellAreaFactory(radio=Radio.gsm, num_cells=1)
area.region = None
BlueShardFactory.create_batch(2, region='CA')
BlueShardFactory.create_batch(3, region='GB')
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1)
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2)
CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2)
CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3)
CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4)
WifiShardFactory.create_batch(5, region='DE')
WifiShardFactory.create_batch(6, region='US')
wifi = WifiShardFactory()
wifi.region = None
self.session.add(RegionStat(region='US', blue=1, wifi=2))
self.session.add(RegionStat(region='TW', wifi=1))
self.session.flush()
update_statregion.delay().get()
stats = self.session.query(RegionStat).all()
self.assertEqual(len(stats), 4)
for stat in stats:
values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi)
if stat.region == 'DE':
self.assertEqual(values, (3, 3, 0, 0, 5))
elif stat.region == 'CA':
self.assertEqual(values, (2, 0, 4, 2, 0))
elif stat.region == 'GB':
self.assertEqual(values, (0, 0, 0, 3, 0))
elif stat.region == 'US':
self.assertEqual(values, (0, 0, 0, 0, 6))
def test_blue(self, geoip_db, http_session, session, source, stats):
now = util.utcnow()
region = GEOCODER.regions_for_mcc(235, metadata=True)[0]
blue1 = BlueShardFactory(samples=10)
blue2 = BlueShardFactory(samples=20)
blue3 = BlueShardFactory.build(region='DE', samples=100)
session.flush()
query = self.model_query(geoip_db,
http_session,
session,
stats,
blues=[blue1, blue2, blue3])
results = source.search(query)
self.check_model_results(results, [region])
best_result = results.best()
assert best_result.region_code == region.code
assert (best_result.score == station_score(blue1, now) +
station_score(blue2, now))
stats.check(counter=[
(self.api_type + '.source', [
'key:test', 'region:none', 'source:internal', 'accuracy:low',
'status:hit'
]),
])
def test_blue(self):
blue = BlueShardFactory()
offset = 0.00001
blues = [
blue,
BlueShardFactory(lat=blue.lat + offset),
BlueShardFactory(lat=blue.lat + offset * 2),
BlueShardFactory(lat=None, lon=None),
]
self.session.flush()
query = self.model_query(blues=blues)
blue_query = query['blue']
blue_query[0]['signal'] = -50
blue_query[1]['signal'] = -150
blue_query[1]['name'] = 'my-beacon'
res = self._call(body=query)
self.check_model_response(res, blue, lat=blue.lat + 0.0000035)
self.check_stats(counter=[
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.result', [
'key:test', 'region:none', 'fallback_allowed:false',
'accuracy:high', 'status:hit', 'source:internal'
]),
(self.metric_type + '.source', [
'key:test', 'region:none', 'source:internal', 'accuracy:high',
'status:hit'
]),
])
def test_blue(self, app, data_queues, session, stats):
blue = BlueShardFactory()
offset = 0.00001
blues = [
blue,
BlueShardFactory(lat=blue.lat + offset),
BlueShardFactory(lat=blue.lat + offset * 2),
BlueShardFactory(lat=None, lon=None),
]
session.flush()
query = self.model_query(blues=blues)
blue_query = query['bluetoothBeacons']
blue_query[0]['signalStrength'] = -50
blue_query[1]['signalStrength'] = -150
blue_query[1]['name'] = 'my-beacon'
res = self._call(app, body=query)
self.check_model_response(res, blue, lat=blue.lat + 0.0000035)
self.check_queue(data_queues, 1)
stats.check(counter=[
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.result', [
'key:test', 'region:none', 'fallback_allowed:false',
'accuracy:high', 'status:hit', 'source:internal'
]),
(self.metric_type + '.source', [
'key:test', 'region:none', 'source:internal', 'accuracy:high',
'status:hit'
]),
])
items = data_queues['update_incoming'].dequeue()
assert (items == [{
'api_key': 'test',
'source': 'query',
'report': {
'bluetoothBeacons': [{
'macAddress': blue_query[0]['macAddress'],
'signalStrength': -50
}, {
'macAddress': blue_query[1]['macAddress'],
'name': 'my-beacon',
}, {
'macAddress': blue_query[2]['macAddress'],
}, {
'macAddress': blue_query[3]['macAddress'],
}],
'fallbacks': {
'ipf': True,
'lacf': True
},
'position': {
'accuracy': BLUE_MIN_ACCURACY,
'latitude': blue.lat + 0.0000035,
'longitude': blue.lon,
'source': 'query',
}
},
}])
def test_blue(self, app, data_queues, session):
# Use manual mac to ensure we only use one shard.
blue1 = BlueShardFactory(mac="000000123456", samples=10)
blue2 = BlueShardFactory(mac="000000abcdef", samples=10)
session.flush()
query = self.model_query(blues=[blue1, blue2])
res = self._call(app, body=query, ip="127.0.0.1")
self.check_response(data_queues, res, blue1)
def test_blue(self):
blue1 = BlueShardFactory(mac='000000123456', samples=10)
blue2 = BlueShardFactory(mac='000000abcdef', samples=10)
self.session.flush()
query = self.model_query(blues=[blue1, blue2])
res = self._call(body=query, ip='127.0.0.1')
self.check_response(res, blue1)
self.check_db_calls(rw=0, ro=2)
def test_blue(self, sync_session):
# Add one network outside the desired area.
BlueShardFactory(lat=46.5743, lon=6.3532, region="FR")
blues = BlueShardFactory.create_batch(1)
sync_session.flush()
self._export(sync_session,
"blue",
self._mac_keys(blues),
restrict=True)
def test_blue_seen(self, app, data_queues, session):
blue = BlueShardFactory()
offset = 0.00002
blues = [
blue,
BlueShardFactory(lat=blue.lat + offset),
]
session.flush()
query = self.model_query(blues=blues)
res = self._call(app, body=query)
self.check_model_response(res, blue, lat=blue.lat + offset / 2)
self.check_queue(data_queues, 0)
def test_blue(self):
blue = BlueShardFactory.build()
res = self._post([{
'lat': blue.lat,
'lon': blue.lon,
'blue': [{
'key': blue.mac,
'age': 3000,
'name': 'beacon',
'signal': -101,
}]},
])
self.assertEqual(res.body, b'')
self.assertEqual(self.queue.size(), 1)
item = self.queue.dequeue()[0]
self.assertEqual(item['api_key'], None)
report = item['report']
position = report['position']
self.assertEqual(position['latitude'], blue.lat)
self.assertEqual(position['longitude'], blue.lon)
blues = item['report']['bluetoothBeacons']
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0]['macAddress'], blue.mac)
self.assertEqual(blues[0]['age'], 3000),
self.assertEqual(blues[0]['name'], 'beacon'),
self.assertEqual(blues[0]['signalStrength'], -101),
def test_blue(self, app, celery):
blue = BlueShardFactory.build()
self._post(app, [
{
'latitude':
blue.lat,
'longitude':
blue.lon,
'bluetoothBeacons': [{
'macAddress': blue.mac,
'age': 3,
'signalStrength': -90,
'name': 'my-beacon',
'xtra_field': 3,
}]
},
])
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item['api_key'] is None
report = item['report']
assert 'timestamp' in report
position = report['position']
assert position['latitude'] == blue.lat
assert position['longitude'] == blue.lon
blues = item['report']['bluetoothBeacons']
assert len(blues) == 1
assert blues[0]['macAddress'] == blue.mac
assert blues[0]['age'] == 3
assert blues[0]['signalStrength'] == -90
assert blues[0]['name'] == 'my-beacon'
assert 'xtra_field' not in blues[0]
def test_blue(self):
blue = BlueShardFactory.build()
self._post([
{
'latitude':
blue.lat,
'longitude':
blue.lon,
'bluetoothBeacons': [{
'macAddress': blue.mac,
'age': 3,
'signalStrength': -90,
'name': 'my-beacon',
'xtra_field': 3,
}]
},
])
self.assertEqual(self.queue.size(), 1)
item = self.queue.dequeue()[0]
self.assertEqual(item['api_key'], None)
report = item['report']
self.assertTrue('timestamp' in report)
position = report['position']
self.assertEqual(position['latitude'], blue.lat)
self.assertEqual(position['longitude'], blue.lon)
blues = item['report']['bluetoothBeacons']
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0]['macAddress'], blue.mac)
self.assertEqual(blues[0]['age'], 3),
self.assertEqual(blues[0]['signalStrength'], -90),
self.assertEqual(blues[0]['name'], 'my-beacon'),
self.assertFalse('xtra_field' in blues[0])
def test_blue_miss(self):
blues = BlueShardFactory.build_batch(2, samples=10)
self.session.flush()
query = self.model_query(blues=blues)
results = self.source.search(query)
self.check_model_results(results, None)
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=[{
'macAddress': blue.mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90
} for blue in blues])
data = OUTBOUND_SCHEMA.deserialize(query.json())
assert (data == {
'bluetoothBeacons': [{
'macAddress': blues[0].mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90,
}, {
'macAddress': blues[1].mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90,
}],
'fallbacks': {
'lacf': True
},
})
def test_blue_miss(self, geoip_db, http_session, session, source):
blues = BlueShardFactory.build_batch(2, samples=10)
session.flush()
query = self.model_query(geoip_db, http_session, session, blues=blues)
results = source.search(query)
self.check_model_results(results, None)
def test_blue(self, app, celery):
blue = BlueShardFactory.build()
res = self._post(app, [{
'lat': blue.lat,
'lon': blue.lon,
'source': '',
'blue': [{
'key': blue.mac,
'age': 3000,
'name': 'beacon',
'signal': -101,
}]},
])
assert res.body == b''
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item['api_key'] is None
report = item['report']
position = report['position']
assert position['latitude'] == blue.lat
assert position['longitude'] == blue.lon
assert 'source' not in position
blues = item['report']['bluetoothBeacons']
assert len(blues) == 1
assert blues[0]['macAddress'] == blue.mac
assert blues[0]['age'] == 3000
assert blues[0]['name'] == 'beacon'
assert blues[0]['signalStrength'] == -101
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key,
'nickname': nickname,
'report': rep} for rep in reports]
self.incoming_queue.enqueue(items)
update_incoming.delay().get()
return reports
def test_blue_not_found(self, app, data_queues, metricsmock, logs):
"""A failed Bluetooth-based lookup emits several metrics."""
blues = BlueShardFactory.build_batch(2)
query = self.model_query(blues=blues)
res = self._call(app, body=query, status=404)
self.check_response(data_queues, res, "not_found")
metricsmock.assert_incr_once(
"request", tags=[self.metric_path, "method:post", "status:404"]
)
metricsmock.assert_incr_once(
self.metric_type + ".request", tags=[self.metric_path, "key:test"]
)
metricsmock.assert_incr_once(
self.metric_type + ".query",
tags=["key:test", "geoip:false", "blue:many", "cell:none", "wifi:none"],
)
metricsmock.assert_incr_once(
self.metric_type + ".result",
tags=["key:test", "accuracy:high", "fallback_allowed:false", "status:miss"],
)
metricsmock.assert_incr_once(
self.metric_type + ".source",
tags=["key:test", "source:internal", "accuracy:high", "status:miss"],
)
assert logs.only_entry["blue_valid"] == 2
def test_blue(self, app, celery):
blue = BlueShardFactory.build()
self._post(app, [{
'latitude': blue.lat,
'longitude': blue.lon,
'bluetoothBeacons': [{
'macAddress': blue.mac,
'age': 3,
'signalStrength': -90,
'name': 'my-beacon',
'xtra_field': 3,
}]},
])
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item['api_key'] is None
report = item['report']
assert 'timestamp' in report
position = report['position']
assert position['latitude'] == blue.lat
assert position['longitude'] == blue.lon
blues = item['report']['bluetoothBeacons']
assert len(blues) == 1
assert blues[0]['macAddress'] == blue.mac
assert blues[0]['age'] == 3
assert blues[0]['signalStrength'] == -90
assert blues[0]['name'] == 'my-beacon'
assert 'xtra_field' not in blues[0]
def test_blue(self, app, celery):
blue = BlueShardFactory.build()
res = self._post(
app,
[{
"lat":
blue.lat,
"lon":
blue.lon,
"source":
"",
"blue": [{
"key": blue.mac,
"age": 3000,
"name": "beacon",
"signal": -101
}],
}],
)
assert res.body == b""
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item["api_key"] is None
report = item["report"]
position = report["position"]
assert position["latitude"] == blue.lat
assert position["longitude"] == blue.lon
assert "source" not in position
blues = item["report"]["bluetoothBeacons"]
assert len(blues) == 1
assert blues[0]["macAddress"] == blue.mac
assert blues[0]["age"] == 3000
assert blues[0]["name"] == "beacon"
assert blues[0]["signalStrength"] == -101
def test_blue_not_found(self, app, data_queues, stats):
blues = BlueShardFactory.build_batch(2)
query = self.model_query(blues=blues)
res = self._call(app, body=query, status=self.not_found.code)
self.check_response(data_queues, res, 'not_found')
stats.check(counter=[
('request', [
self.metric_path, 'method:post',
'status:%s' % self.not_found.code
]),
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.query', [
'key:test', 'region:none', 'geoip:false', 'blue:many',
'cell:none', 'wifi:none'
]),
(self.metric_type + '.result', 'fallback_allowed:false',
['key:test', 'region:none', 'accuracy:high', 'status:miss']),
(self.metric_type + '.source', [
'key:test', 'region:none', 'source:internal', 'accuracy:high',
'status:miss'
]),
],
timer=[
('request', [self.metric_path, 'method:post']),
])
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
query = Query(
blue=[
{
"macAddress": blue.mac,
"age": 1500,
"name": "beacon",
"signalStrength": -90,
}
for blue in blues
]
)
data = self._call(query.json())
assert data == {
"bluetoothBeacons": [
{
"macAddress": blues[0].mac,
"age": 1500,
"name": "beacon",
"signalStrength": -90,
},
{
"macAddress": blues[1].mac,
"age": 1500,
"name": "beacon",
"signalStrength": -90,
},
],
"fallbacks": {"lacf": True},
}
def test_blue(self):
blue = BlueShardFactory.build()
self._post([{
'latitude': blue.lat,
'longitude': blue.lon,
'bluetoothBeacons': [{
'macAddress': blue.mac,
'age': 3,
'signalStrength': -90,
'name': 'my-beacon',
'xtra_field': 3,
}]},
])
self.assertEqual(self.queue.size(), 1)
item = self.queue.dequeue()[0]
self.assertEqual(item['api_key'], None)
report = item['report']
self.assertTrue('timestamp' in report)
position = report['position']
self.assertEqual(position['latitude'], blue.lat)
self.assertEqual(position['longitude'], blue.lon)
blues = item['report']['bluetoothBeacons']
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0]['macAddress'], blue.mac)
self.assertEqual(blues[0]['age'], 3),
self.assertEqual(blues[0]['signalStrength'], -90),
self.assertEqual(blues[0]['name'], 'my-beacon'),
self.assertFalse('xtra_field' in blues[0])
def test_get_blue(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=self.blue_model_query(blues))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:miss']),
])
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=[{
'mac': blue.mac,
'age': 1500,
'name': 'beacon',
'signal': -90
} for blue in blues])
data = self.schema.deserialize(query.internal_query())
self.assertEqual(
data, {
'bluetoothBeacons': [{
'macAddress': blues[0].mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90,
}, {
'macAddress': blues[1].mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90,
}],
'fallbacks': {
'lacf': True
},
})
def test_get_blue(self):
blues = BlueShardFactory.build_batch(2)
query = self._query(blue=self.blue_model_query(blues))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:miss']),
])
def test_blue(self, app, celery):
blue = BlueShardFactory.build()
self._post(
app,
[{
"latitude":
blue.lat,
"longitude":
blue.lon,
"bluetoothBeacons": [{
"macAddress": blue.mac,
"age": 3,
"signalStrength": -90,
"name": "my-beacon",
"xtra_field": 3,
}],
}],
)
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item["api_key"] is None
report = item["report"]
assert "timestamp" in report
position = report["position"]
assert position["latitude"] == blue.lat
assert position["longitude"] == blue.lon
blues = item["report"]["bluetoothBeacons"]
assert len(blues) == 1
assert blues[0]["macAddress"] == blue.mac
assert blues[0]["age"] == 3
assert blues[0]["signalStrength"] == -90
assert blues[0]["name"] == "my-beacon"
assert "xtra_field" not in blues[0]
def test_blue(self, app, celery):
blue = BlueShardFactory.build()
res = self._post(app, [{
'lat': blue.lat,
'lon': blue.lon,
'source': '',
'blue': [{
'key': blue.mac,
'age': 3000,
'name': 'beacon',
'signal': -101,
}]},
])
assert res.body == b''
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
assert item['api_key'] is None
report = item['report']
position = report['position']
assert position['latitude'] == blue.lat
assert position['longitude'] == blue.lon
assert 'source' not in position
blues = item['report']['bluetoothBeacons']
assert len(blues) == 1
assert blues[0]['macAddress'] == blue.mac
assert blues[0]['age'] == 3000
assert blues[0]['name'] == 'beacon'
assert blues[0]['signalStrength'] == -101
def test_get_blue(self, cache, stats):
blues = BlueShardFactory.build_batch(2)
query = self._query(blue=self.blue_model_query(blues))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1, ['status:miss']),
])
def test_blue_duplicates(self):
blue = BlueShardFactory.build()
query = Query(blue=[
{'mac': blue.mac, 'signal': -90},
{'mac': blue.mac, 'signal': -82},
{'mac': blue.mac, 'signal': -85},
])
self.assertEqual(len(query.blue), 0)
def test_blue_duplicates(self):
blue = BlueShardFactory.build()
query = Query(blue=[
{'macAddress': blue.mac, 'signalStrength': -90},
{'macAddress': blue.mac, 'signalStrength': -82},
{'macAddress': blue.mac, 'signalStrength': -85},
])
assert len(query.blue) == 0
def test_check_one_blue(self, geoip_db, http_session,
session, source, stats):
blue = BlueShardFactory.build()
query = self.model_query(
geoip_db, http_session, session, stats,
blues=[blue])
self.check_should_search(source, query, False)
def test_get_blue(self, cache, stats):
blues = BlueShardFactory.build_batch(2)
query = self._query(blue=self.blue_model_query(blues))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1,
[self.fallback_tag, 'status:miss']),
])
def test_blue_miss(self, geoip_db, http_session, session, source):
"""Unknown Bluetooth stations fail to determine the region."""
blues = BlueShardFactory.build_batch(2, samples=10)
session.flush()
query = self.model_query(geoip_db, http_session, session, blues=blues)
results = source.search(query)
self.check_model_results(results, None)
def test_blue_seen(self, app, data_queues, session):
"""If a query contains no new data, it is not queued for further processing.
This fails occasionally, see issue #921.
"""
self.check_queue(data_queues, 0)
blue = BlueShardFactory()
offset = 0.00002
blues = [blue, BlueShardFactory(lat=blue.lat + offset)]
session.flush()
self.check_queue(data_queues, 0)
query = self.model_query(blues=blues)
res = self._call(app, body=query)
self.check_model_response(res, blue, lat=blue.lat + offset / 2)
if data_queues["update_incoming"].size():
items = data_queues["update_incoming"].dequeue()
pytest.fail(("Query added a report to the update_incoming queue."
"\nQuery:\n{}\nReport:\n{}").format(query, items))
def test_empty_blue_entry(self):
blue = BlueShardFactory.build()
data = SUBMIT_V0_SCHEMA.deserialize(
{"items": [{
"lat": blue.lat,
"lon": blue.lon,
"blue": [{}]
}]})
assert data == {"items": []}
def test_set_blue(self, cache, metricsmock):
blues = BlueShardFactory.build_batch(2)
blue = blues[0]
query = self._query(blue=self.blue_model_query(blues))
result = ExternalResult(blue.lat, blue.lon, blue.radius, None)
cache.set(query, result)
assert cache.get(query) == result
metricsmock.assert_incr_once("locate.fallback.cache",
tags=[self.fallback_tag, "status:hit"])
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
blue_key=None, cell_mcc=None, wifi_key=None,
api_key='test', lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key, 'report': rep} for rep in reports]
self.queue.enqueue(items)
return reports
def test_blue_miss(self, geoip_db, http_session,
session, source, stats):
blues = BlueShardFactory.build_batch(2, samples=10)
session.flush()
query = self.model_query(
geoip_db, http_session, session, stats,
blues=blues)
results = source.search(query)
self.check_model_results(results, None)
def test_set_blue(self):
blues = BlueShardFactory.build_batch(2)
blue = blues[0]
query = Query(blue=self.blue_model_query(blues))
result = ExternalResult(blue.lat, blue.lon, blue.radius, None)
self.cache.set(query, result)
self.assertEqual(self.cache.get(query), result)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:hit']),
])
def test_set_blue(self, cache, stats):
blues = BlueShardFactory.build_batch(2)
blue = blues[0]
query = self._query(blue=self.blue_model_query(blues))
result = ExternalResult(blue.lat, blue.lon, blue.radius, None)
cache.set(query, result)
assert cache.get(query) == result
stats.check(counter=[
('locate.fallback.cache', 1, 1,
[self.fallback_tag, 'status:hit']),
])
def test_blue(self):
now = util.utcnow()
region = GEOCODER.regions_for_mcc(235, metadata=True)[0]
blue1 = BlueShardFactory(samples=10)
blue2 = BlueShardFactory(samples=20)
blue3 = BlueShardFactory.build(region='DE', samples=100)
self.session.flush()
query = self.model_query(blues=[blue1, blue2, blue3])
results = self.source.search(query)
self.check_model_results(results, [region])
best_result = results.best()
self.assertEqual(best_result.region_code, region.code)
self.assertAlmostEqual(
best_result.score, blue1.score(now) + blue2.score(now), 4)
self.check_stats(counter=[
(self.api_type + '.source',
['key:test', 'region:none', 'source:internal',
'accuracy:low', 'status:hit']),
])
def test_many(self):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(3)
self._make_query(blue=blues, cell=cells, wifi=wifis, ip=self.london_ip)
self.check_stats(total=1, counter=[
('locate.query',
['key:key', 'region:GB',
'blue:many', 'cell:many', 'wifi:many']),
])
def test_blue_better(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=[
{'mac': blues[0].mac, 'signal': -90, 'name': 'my-beacon'},
{'mac': blues[0].mac, 'signal': -82},
{'mac': blues[0].mac, 'signal': -85},
{'mac': blues[1].mac, 'signal': -70},
])
self.assertEqual(len(query.blue), 2)
self.assertEqual(
set([blue.signal for blue in query.blue]), set([-70, -82]))
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
macs = [blue.mac for blue in blues]
query = Query(blue=self.blue_model_query(blues))
self.assertEqual(len(query.blue), 2)
self.assertEqual(query.expected_accuracy, DataAccuracy.high)
for blue in query.blue:
self.assertEqual(blue.age, 10)
self.assertEqual(blue.signal, -85)
self.assertTrue(blue.mac in macs)
def test_blue_better(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=[
{'macAddress': blues[0].mac, 'signalStrength': -90,
'name': 'my-beacon'},
{'macAddress': blues[0].mac, 'signalStrength': -82},
{'macAddress': blues[0].mac, 'signalStrength': -85},
{'macAddress': blues[1].mac, 'signalStrength': -70},
])
assert len(query.blue) == 2
assert (set([blue.signalStrength for blue in query.blue]) ==
set([-70, -82]))
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
macs = [blue.mac for blue in blues]
query = Query(blue=self.blue_model_query(blues))
assert len(query.blue) == 2
assert query.expected_accuracy is DataAccuracy.high
for blue in query.blue:
assert blue.age == 10
assert blue.signalStrength == -85
assert blue.macAddress in macs
def test_one(self, geoip_db, stats):
blues = BlueShardFactory.build_batch(1)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(1)
self._make_query(
geoip_db, stats,
blue=blues, cell=cells, wifi=wifis, ip=self.london_ip)
stats.check(total=1, counter=[
('locate.query',
['key:test', 'region:GB',
'blue:one', 'cell:one', 'wifi:one']),
])
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=[
{'mac': blue.mac, 'signal': -90, 'name': 'my'} for blue in blues])
data = self.schema.deserialize(query.internal_query())
self.assertEqual(data, {
'bluetoothBeacons': [{
'macAddress': blues[0].mac,
'signalStrength': -90,
'name': 'my',
}, {
'macAddress': blues[1].mac,
'signalStrength': -90,
'name': 'my',
}],
'fallbacks': {'lacf': True},
})
def test_get_mixed(self):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
self.assertEqual(self.cache.get(query), None)
query = Query(blue=self.blue_model_query(blues),
cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
query = Query(blue=self.blue_model_query(blues),
wifi=self.wifi_model_query(wifis))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 3, 1, ['status:bypassed']),
])
def test_blue(self):
blues = BlueShardFactory.build_batch(2)
query = Query(blue=[
{'macAddress': blue.mac, 'age': 1500, 'name': 'beacon',
'signalStrength': -90}
for blue in blues])
data = self._call(query.json())
assert (data == {
'bluetoothBeacons': [{
'macAddress': blues[0].mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90,
}, {
'macAddress': blues[1].mac,
'age': 1500,
'name': 'beacon',
'signalStrength': -90,
}],
'fallbacks': {'lacf': True},
})
def test_get_mixed(self, cache, stats):
blues = BlueShardFactory.build_batch(2)
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
assert cache.get(query) is None
query = self._query(blue=self.blue_model_query(blues),
cell=self.cell_model_query(cells))
assert cache.get(query) is None
query = self._query(blue=self.blue_model_query(blues),
wifi=self.wifi_model_query(wifis))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 3, 1,
[self.fallback_tag, 'status:bypassed']),
])
def test_blue_not_found(self):
blues = BlueShardFactory.build_batch(2)
query = self.model_query(blues=blues)
res = self._call(body=query, status=self.not_found.code)
self.check_response(res, 'not_found')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post',
'status:%s' % self.not_found.code]),
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.query',
['key:test', 'region:none',
'geoip:false', 'blue:many', 'cell:none', 'wifi:none']),
(self.metric_type + '.result', 'fallback_allowed:false',
['key:test', 'region:none', 'accuracy:high', 'status:miss']),
(self.metric_type + '.source',
['key:test', 'region:none', 'source:internal',
'accuracy:high', 'status:miss']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
def test_bluetooth(self, app, celery):
blue = BlueShardFactory.build()
self._post(app, [{
'position': {
'latitude': blue.lat,
'longitude': blue.lon,
},
'bluetoothBeacons': [{
'macAddress': blue.mac,
'name': 'my-beacon',
'age': 3,
'signalStrength': -90,
'xtra_field': 4,
}, {
'name': 'beacon-2',
'signalStrength': -92,
}],
'wifiAccessPoints': [{
'signalStrength': -52,
}]},
])
assert self.queue(celery).size() == 1
item = self.queue(celery).dequeue()[0]
report = item['report']
assert 'timestamp' in report
position = report['position']
assert position['latitude'] == blue.lat
assert position['longitude'] == blue.lon
blues = report['bluetoothBeacons']
assert len(blues) == 1
assert blues[0]['macAddress'] == blue.mac
assert blues[0]['age'] == 3
assert blues[0]['name'] == 'my-beacon'
assert blues[0]['signalStrength'] == -90
assert 'xtra_field' not in blues[0]
wifis = report['wifiAccessPoints']
assert len(wifis) == 0
def test_bluetooth(self):
blue = BlueShardFactory.build()
self._post([{
'position': {
'latitude': blue.lat,
'longitude': blue.lon,
},
'bluetoothBeacons': [{
'macAddress': blue.mac,
'name': 'my-beacon',
'age': 3,
'signalStrength': -90,
'xtra_field': 4,
}, {
'name': 'beacon-2',
'signalStrength': -92,
}],
'wifiAccessPoints': [{
'signalStrength': -52,
}]},
])
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
report = item['report']
self.assertTrue('timestamp' in report)
position = report['position']
self.assertEqual(position['latitude'], blue.lat)
self.assertEqual(position['longitude'], blue.lon)
blues = report['bluetoothBeacons']
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0]['macAddress'], blue.mac)
self.assertEqual(blues[0]['age'], 3),
self.assertEqual(blues[0]['name'], 'my-beacon'),
self.assertEqual(blues[0]['signalStrength'], -90),
self.assertFalse('xtra_field' in blues[0])
wifis = report['wifiAccessPoints']
self.assertEqual(len(wifis), 0)
def test_blue(self, geoip_db, http_session,
session, source, stats):
now = util.utcnow()
region = GEOCODER.regions_for_mcc(235, metadata=True)[0]
blue1 = BlueShardFactory(samples=10)
blue2 = BlueShardFactory(samples=20)
blue3 = BlueShardFactory.build(region='DE', samples=100)
session.flush()
query = self.model_query(
geoip_db, http_session, session, stats,
blues=[blue1, blue2, blue3])
results = source.search(query)
self.check_model_results(results, [region])
best_result = results.best()
assert best_result.region_code == region.code
assert (best_result.score ==
station_score(blue1, now) + station_score(blue2, now))
stats.check(counter=[
(self.api_type + '.source',
['key:test', 'region:none', 'source:internal',
'accuracy:low', 'status:hit']),
])
