def test_wifi_permanent_blocked(self, geoip_db, http_session, session,
source, stats):
now = util.utcnow()
last_week = now - timedelta(days=7)
three_months = now - timedelta(days=90)
four_months = now - timedelta(days=120)
wifi = WifiShardFactory(radius=200)
wifi2 = WifiShardFactory(lat=wifi.lat,
lon=wifi.lon + 0.00001,
radius=300,
created=four_months,
modified=now,
block_first=three_months.date(),
block_last=last_week.date(),
block_count=4)
session.flush()
query = self.model_query(geoip_db,
http_session,
session,
stats,
wifis=[wifi, wifi2])
results = source.search(query)
self.check_model_results(results, None)
def test_wifi(self, geoip_db, http_session, session, source, metricsmock):
now = util.utcnow()
region = GEOCODER.regions_for_mcc(235, metadata=True)[0]
wifi1 = WifiShardFactory(samples=10)
wifi2 = WifiShardFactory(samples=20)
wifi3 = WifiShardFactory.build(region="DE", samples=100)
session.flush()
query = self.model_query(
geoip_db, http_session, session, wifis=[wifi1, wifi2, wifi3]
)
results = source.search(query)
self.check_model_results(results, [region])
best_result = results.best()
assert best_result.region_code == region.code
assert best_result.score == station_score(wifi1, now) + station_score(
wifi2, now
)
assert metricsmock.has_record(
"incr",
self.api_type + ".source",
value=1,
tags=[
"key:test",
"region:none",
"source:internal",
"accuracy:low",
"status:hit",
],
)
def test_check_invalid_wifi(self):
wifi = WifiShardFactory.build()
malformed_wifi = WifiShardFactory.build()
malformed_wifi.mac = 'abcd'
query = self.model_query(wifis=[wifi, malformed_wifi])
self.check_should_search(query, False)
def test_wifi(self):
wifi = WifiShardFactory()
offset = 0.00001
wifis = [
wifi,
WifiShardFactory(lat=wifi.lat + offset),
WifiShardFactory(lat=wifi.lat + offset * 2),
WifiShardFactory(lat=None, lon=None),
]
self.session.flush()
query = self.model_query(wifis=wifis)
wifi_query = query['wifi']
wifi_query[0]['channel'] = 6
wifi_query[0]['signal'] = -50
wifi_query[1]['frequency'] = 2437
wifi_query[2]['signal'] = -130
wifi_query[2]['signalToNoiseRatio'] = 13
wifi_query[3]['ssid'] = 'my-wifi'
res = self._call(body=query)
self.check_model_response(res, wifi, lat=wifi.lat + 0.000005)
self.check_stats(counter=[
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.result', [
'key:test', 'region:none', 'fallback_allowed:false',
'accuracy:high', 'status:hit', 'source:internal'
]),
(self.metric_type + '.source', [
'key:test', 'region:none', 'source:internal', 'accuracy:high',
'status:hit'
]),
])
def test_set_wifi_inconsistent(self):
wifis1 = WifiShardFactory.build_batch(2)
self.cache.set(
Query(wifi=self.wifi_model_query(wifis1)),
ExternalResult(wifis1[0].lat, wifis1[0].lon, 100, None))
# similar lat/lon, worse accuracy
wifis2 = WifiShardFactory.build_batch(
2, lat=wifis1[0].lat + 0.0001, lon=wifis1[0].lon)
self.cache.set(
Query(wifi=self.wifi_model_query(wifis2)),
ExternalResult(wifis2[0].lat, wifis2[0].lon, 200, None))
# check combined query, avg lat/lon, max accuracy
query = Query(wifi=self.wifi_model_query(wifis1 + wifis2))
self.assertEqual(
self.cache.get(query),
((wifis1[0].lat + wifis2[0].lat) / 2, wifis1[0].lon, 200, None))
# different lat/lon
wifis3 = WifiShardFactory.build_batch(2, lat=wifis1[0].lat + 10.0)
self.cache.set(
Query(wifi=self.wifi_model_query(wifis3)),
ExternalResult(wifis3[0].lat, wifis3[0].lon, 300, None))
# check combined query, inconsistent result
query = Query(wifi=self.wifi_model_query(wifis1 + wifis2 + wifis3))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:hit']),
('locate.fallback.cache', 1, 1, ['status:inconsistent']),
])
def test_region(self):
obs = []
obs_factory = WifiObservationFactory
wifi1 = WifiShardFactory(lat=46.2884, lon=6.77, region='FR')
obs.extend(
obs_factory.create_batch(
5,
key=wifi1.mac,
lat=wifi1.lat,
lon=wifi1.lon + 0.05,
))
wifi2 = WifiShardFactory(lat=46.2884, lon=7.4, region='FR')
obs.extend(
obs_factory.create_batch(
5,
key=wifi2.mac,
lat=wifi2.lat,
lon=wifi2.lon + 0.05,
))
self.session.commit()
self._queue_and_update(obs)
# position is really not in FR anymore, but still close enough
# to not re-trigger region determination
wifi1 = self.session.query(wifi1.__class__).get(wifi1.mac)
self.assertEqual(wifi1.block_count, 0)
self.assertEqual(wifi1.region, 'FR')
wifi2 = self.session.query(wifi2.__class__).get(wifi2.mac)
self.assertEqual(wifi1.block_count, 0)
self.assertEqual(wifi2.region, 'CH')
def test_wifi(self, geoip_db, http_session, session, source, stats):
now = util.utcnow()
region = GEOCODER.regions_for_mcc(235, metadata=True)[0]
wifi1 = WifiShardFactory(samples=10)
wifi2 = WifiShardFactory(samples=20)
wifi3 = WifiShardFactory.build(region='DE', samples=100)
session.flush()
query = self.model_query(geoip_db,
http_session,
session,
stats,
wifis=[wifi1, wifi2, wifi3])
results = source.search(query)
self.check_model_results(results, [region])
best_result = results.best()
assert best_result.region_code == region.code
assert (best_result.score == station_score(wifi1, now) +
station_score(wifi2, now))
stats.check(counter=[
(self.api_type + '.source', [
'key:test', 'region:none', 'source:internal', 'accuracy:low',
'status:hit'
]),
])
def test_set_wifi_inconsistent(self):
wifis1 = WifiShardFactory.build_batch(2)
self.cache.set(self._query(wifi=self.wifi_model_query(wifis1)),
ExternalResult(wifis1[0].lat, wifis1[0].lon, 100, None))
# similar lat/lon, worse accuracy
wifis2 = WifiShardFactory.build_batch(2,
lat=wifis1[0].lat + 0.0001,
lon=wifis1[0].lon)
self.cache.set(self._query(wifi=self.wifi_model_query(wifis2)),
ExternalResult(wifis2[0].lat, wifis2[0].lon, 200, None))
# check combined query, avg lat/lon, max accuracy
query = self._query(wifi=self.wifi_model_query(wifis1 + wifis2))
cached = self.cache.get(query)
self.assertAlmostEqual(cached[0], (wifis1[0].lat + wifis2[0].lat) / 2)
self.assertAlmostEqual(cached[1], wifis1[0].lon)
self.assertAlmostEqual(cached[2], 205.56, 2)
self.assertTrue(cached[3] is None)
# different lat/lon
wifis3 = WifiShardFactory.build_batch(2, lat=wifis1[0].lat + 10.0)
self.cache.set(self._query(wifi=self.wifi_model_query(wifis3)),
ExternalResult(wifis3[0].lat, wifis3[0].lon, 300, None))
# check combined query, inconsistent result
query = self._query(wifi=self.wifi_model_query(wifis1 + wifis2 +
wifis3))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:hit']),
('locate.fallback.cache', 1, 1, ['status:inconsistent']),
])
def test_set_wifi_inconsistent(self, cache, stats):
wifis1 = WifiShardFactory.build_batch(2)
cache.set(self._query(wifi=self.wifi_model_query(wifis1)),
ExternalResult(wifis1[0].lat, wifis1[0].lon, 100, None))
# similar lat/lon, worse accuracy
wifis2 = WifiShardFactory.build_batch(2,
lat=wifis1[0].lat + 0.0001,
lon=wifis1[0].lon)
cache.set(self._query(wifi=self.wifi_model_query(wifis2)),
ExternalResult(wifis2[0].lat, wifis2[0].lon, 200, None))
# check combined query, avg lat/lon, max accuracy
query = self._query(wifi=self.wifi_model_query(wifis1 + wifis2))
cached = cache.get(query)
assert cached[0] == (wifis1[0].lat + wifis2[0].lat) / 2.0
assert cached[1] == wifis1[0].lon
assert round(cached[2], 2) == 205.56
assert cached[3] is None
# different lat/lon
wifis3 = WifiShardFactory.build_batch(2, lat=wifis1[0].lat + 10.0)
cache.set(self._query(wifi=self.wifi_model_query(wifis3)),
ExternalResult(wifis3[0].lat, wifis3[0].lon, 300, None))
# check combined query, inconsistent result
query = self._query(wifi=self.wifi_model_query(wifis1 + wifis2 +
wifis3))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1, ['status:hit']),
('locate.fallback.cache', 1, 1, ['status:inconsistent']),
])
def test_weight(self, geoip_db, http_session, session, source):
wifi1 = WifiShardFactory.build()
wifis = []
for i in range(4):
wifis.append(
WifiShardFactory(
lat=wifi1.lat + i * 0.0001, lon=wifi1.lon + i * 0.00012
)
)
session.flush()
query = self.model_query(geoip_db, http_session, session, wifis=wifis)
query.wifi[0].signalStrength = -10
query.wifi[0].age = 0
query.wifi[1].signalStrength = -40
query.wifi[1].age = 8000
query.wifi[2].signalStrength = -70
query.wifi[2].age = 16000
query.wifi[3].signalStrength = -100
results = source.search(query)
result = results.best()
assert round(result.lat, 7) == wifi1.lat + 0.0000009
assert round(result.lon, 7) == wifi1.lon + 0.0000006
assert round(result.accuracy, 2) == 39.51
def test_cluster_score_over_size(self):
now = util.utcnow()
yesterday = now - timedelta(days=1)
last_week = now - timedelta(days=7)
three_months = now - timedelta(days=90)
four_months = now - timedelta(days=120)
wifi11 = WifiShardFactory(
samples=20, created=last_week, modified=yesterday)
wifi12 = WifiShardFactory(
lat=wifi11.lat + 0.0003, lon=wifi11.lon,
samples=30, created=yesterday, modified=now)
wifi13 = WifiShardFactory(
lat=wifi11.lat - 0.0003, lon=wifi11.lon,
samples=10, created=yesterday, modified=now)
wifi21 = WifiShardFactory(
lat=wifi11.lat + 1.0, lon=wifi11.lon + 1.0,
samples=40, created=four_months, modified=three_months)
wifi22 = WifiShardFactory(
lat=wifi21.lat, lon=wifi21.lon,
samples=50, created=three_months, modified=last_week)
self.session.flush()
query = self.model_query(
wifis=[wifi11, wifi12, wifi13, wifi21, wifi22])
results = self.source.search(query)
self.check_model_results(results, [wifi11, wifi21])
best_result = results.best(query.expected_accuracy)
self.assertAlmostEqual(best_result.lat, wifi21.lat, 7)
self.assertAlmostEqual(best_result.lon, wifi21.lon, 7)
self.assertAlmostEqual(
best_result.score, wifi21.score(now) + wifi22.score(now), 4)
def test_check_invalid_wifi(self):
wifi = WifiShardFactory.build()
malformed_wifi = WifiShardFactory.build()
malformed_wifi.mac = 'abcd'
query = self.model_query(wifis=[wifi, malformed_wifi])
self.check_should_search(query, False)
def test_top_results_in_noisy_cluster(self):
now = util.utcnow()
# all these should wind up in the same cluster since
# the WiFis are spaced in increments of (+0.1m, +0.12m)
wifi1 = WifiShardFactory.build()
wifis = []
for i in range(0, MAX_WIFIS_IN_CLUSTER + 10):
wifis.append(
WifiShardFactory(lat=wifi1.lat + i * 0.000001,
lon=wifi1.lon + i * 0.0000012,
samples=100 - i))
self.session.flush()
# calculate expected result
score = sum([wifi.score(now) for wifi in wifis])
query = self.model_query(wifis=wifis)
for i, entry in enumerate(query.wifi):
entry.signal = -50 - i
results = self.source.search(query)
result = results.best()
self.assertAlmostEqual(result.lat, wifi1.lat, 4)
self.assertAlmostEqual(result.lon, wifi1.lon, 4)
self.assertAlmostEqual(result.score, score, 4)
def test_update(self):
utcnow = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
# first wifi
wifi1 = WifiShardFactory(lat=None, lon=None, samples=3)
new_pos = WifiShardFactory.build()
mac1, lat1, lon1 = (wifi1.mac, new_pos.lat, new_pos.lon)
obs.extend([
obs_factory(lat=lat1, lon=lon1, key=mac1),
obs_factory(lat=lat1 + 0.002, lon=lon1 + 0.003, key=mac1),
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, key=mac1),
])
# second wifi
wifi2 = WifiShardFactory(lat=lat1 + 1.0,
lon=lon1 + 1.0,
max_lat=lat1 + 1.0,
min_lat=lat1 + 0.999,
max_lon=lon1 + 1.0,
min_lon=None,
radius=20,
samples=2,
created=utcnow - timedelta(10),
modified=utcnow - timedelta(10))
mac2, lat2, lon2 = (wifi2.mac, wifi2.lat, wifi2.lon)
obs.extend([
obs_factory(lat=lat2 + 0.002, lon=lon2 + 0.004, key=mac2),
obs_factory(lat=lat2 + 0.002, lon=lon2 + 0.004, key=mac2),
])
self.session.commit()
self._queue_and_update(obs)
shard = WifiShard.shard_model(mac1)
found = self.session.query(shard).filter(shard.mac == mac1).one()
self.assertAlmostEqual(found.lat, lat1 + 0.002)
self.assertAlmostEqual(found.max_lat, lat1 + 0.004)
self.assertAlmostEqual(found.min_lat, lat1)
self.assertAlmostEqual(found.lon, lon1 + 0.003)
self.assertAlmostEqual(found.max_lon, lon1 + 0.006)
self.assertAlmostEqual(found.min_lon, lon1)
self.assertEqual(found.modified.date(), utcnow.date())
self.assertEqual(found.radius, 304)
self.assertEqual(found.region, 'GB')
self.assertEqual(found.samples, 6)
shard = WifiShard.shard_model(mac2)
found = self.session.query(shard).filter(shard.mac == mac2).one()
self.assertAlmostEqual(found.lat, lat2 + 0.001)
self.assertAlmostEqual(found.max_lat, lat2 + 0.002)
self.assertAlmostEqual(found.min_lat, lat2 - 0.001)
self.assertAlmostEqual(found.lon, lon2 + 0.002)
self.assertAlmostEqual(found.max_lon, lon2 + 0.004)
self.assertAlmostEqual(found.min_lon, lon2)
self.assertEqual(found.created.date(), utcnow.date() - timedelta(10))
self.assertEqual(found.modified.date(), utcnow.date())
self.assertEqual(found.radius, 260)
self.assertEqual(found.region, 'GB')
self.assertEqual(found.samples, 4)
def test_wifi_no_position(self):
wifi = WifiShardFactory()
wifi2 = WifiShardFactory(lat=wifi.lat, lon=wifi.lon)
wifi3 = WifiShardFactory(lat=None, lon=wifi.lon, radius=None)
self.session.flush()
query = self.model_query(wifis=[wifi, wifi2, wifi3])
results = self.source.search(query)
self.check_model_results(results, [wifi])
def test_set_wifi_inconsistent(self, cache, metricsmock):
wifis1 = WifiShardFactory.build_batch(2)
cache.set(
self._query(wifi=self.wifi_model_query(wifis1)),
ExternalResult(wifis1[0].lat, wifis1[0].lon, 100, None),
)
# similar lat/lon, worse accuracy
wifis2 = WifiShardFactory.build_batch(
2, lat=wifis1[0].lat + 0.0001, lon=wifis1[0].lon
)
cache.set(
self._query(wifi=self.wifi_model_query(wifis2)),
ExternalResult(wifis2[0].lat, wifis2[0].lon, 200, None),
)
# check combined query, avg lat/lon, max accuracy
query = self._query(wifi=self.wifi_model_query(wifis1 + wifis2))
cached = cache.get(query)
assert cached[0] == (wifis1[0].lat + wifis2[0].lat) / 2.0
assert cached[1] == wifis1[0].lon
assert round(cached[2], 2) == 205.56
assert cached[3] is None
# different lat/lon
wifis3 = WifiShardFactory.build_batch(2, lat=wifis1[0].lat + 10.0)
cache.set(
self._query(wifi=self.wifi_model_query(wifis3)),
ExternalResult(wifis3[0].lat, wifis3[0].lon, 300, None),
)
# check combined query, inconsistent result
query = self._query(wifi=self.wifi_model_query(wifis1 + wifis2 + wifis3))
assert cache.get(query) is None
assert (
len(
metricsmock.filter_records(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:hit"],
)
)
== 1
)
assert (
len(
metricsmock.filter_records(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:inconsistent"],
)
)
== 1
)
def test_check_invalid_wifi(self, geoip_db, http_session, session, source):
wifi = WifiShardFactory.build()
malformed_wifi = WifiShardFactory.build()
malformed_wifi.mac = "abcd"
query = self.model_query(
geoip_db, http_session, session, wifis=[wifi, malformed_wifi]
)
self.check_should_search(source, query, False)
def test_wifi(self):
wifi1 = WifiShardFactory(mac='000000123456', samples=10)
wifi2 = WifiShardFactory(mac='000000abcdef', samples=10)
self.session.flush()
query = self.model_query(wifis=[wifi1, wifi2])
res = self._call(body=query, ip='127.0.0.1')
self.check_response(res, wifi1)
self.check_db_calls(rw=0, ro=2)
def test_wifi(self, app, data_queues, session):
# Use manual mac to ensure we only use one shard.
wifi1 = WifiShardFactory(mac="000000123456", samples=10)
wifi2 = WifiShardFactory(mac="000000abcdef", samples=10)
session.flush()
query = self.model_query(wifis=[wifi1, wifi2])
res = self._call(app, body=query, ip="127.0.0.1")
self.check_response(data_queues, res, wifi1)
def test_cluster_score_over_size(self, geoip_db, http_session, session, source):
now = util.utcnow()
yesterday = now - timedelta(days=1)
last_week = now - timedelta(days=7)
three_months = now - timedelta(days=90)
four_months = now - timedelta(days=120)
wifi11 = WifiShardFactory(samples=20, created=last_week, modified=yesterday)
wifi12 = WifiShardFactory(
lat=wifi11.lat + 0.00003,
lon=wifi11.lon,
samples=30,
created=yesterday,
modified=now,
)
wifi13 = WifiShardFactory(
lat=wifi11.lat - 0.00003,
lon=wifi11.lon,
samples=10,
created=yesterday,
modified=now,
)
wifi21 = WifiShardFactory(
lat=wifi11.lat + 1.0,
lon=wifi11.lon + 1.0,
samples=40,
created=four_months,
modified=three_months,
)
wifi22 = WifiShardFactory(
lat=wifi21.lat,
lon=wifi21.lon,
samples=50,
created=three_months,
modified=last_week,
)
session.flush()
query = self.model_query(
geoip_db,
http_session,
session,
wifis=[wifi11, wifi12, wifi13, wifi21, wifi22],
)
results = source.search(query)
assert len(results) == 2
best_result = results.best()
assert round(best_result.lat, 7) == round(wifi21.lat, 7)
assert round(best_result.lon, 7) == round(wifi21.lon, 7)
assert round(best_result.accuracy, 2) == 10.0
assert round(best_result.score, 2) == round(
station_score(wifi21, now) + station_score(wifi22, now), 2
)
other_result = [res for res in results if res.score < best_result.score][0]
assert round(other_result.lat, 4) == round(wifi11.lat, 4)
assert round(other_result.lon, 4) == round(wifi11.lon, 4)
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = WifiShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
queue_reports.delay(reports=reports, api_key=api_key,
nickname=nickname).get()
return reports
def test_check_invalid_wifi(self, geoip_db, http_session,
session, source, stats):
wifi = WifiShardFactory.build()
malformed_wifi = WifiShardFactory.build()
malformed_wifi.mac = 'abcd'
query = self.model_query(
geoip_db, http_session, session, stats,
wifis=[wifi, malformed_wifi])
self.check_should_search(source, query, False)
def test_ignore_outlier(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon)
wifis[0].lat = wifi.lat + 0.0001
wifis[1].lat = wifi.lat + 0.0002
wifis[2].lat = wifi.lat + 1.0
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis)
result = self.source.search(query)
self.check_model_result(result, wifi, lat=wifi.lat + 0.0001)
def test_ignore_outlier(self, geoip_db, http_session, session, source):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon, radius=5)
wifis[0].lat = wifi.lat + 0.00001
wifis[1].lat = wifi.lat + 0.00002
wifis[2].lat = wifi.lat + 1.0
session.flush()
query = self.model_query(geoip_db, http_session, session, wifis=[wifi] + wifis)
results = source.search(query)
self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001)
assert round(results.best().score, 4) == 0.15
def test_wifi_permanent_blocked(self):
wifi = WifiShardFactory(radius=200)
wifi2 = WifiShardFactory(lat=wifi.lat,
lon=wifi.lon + 0.00001,
radius=300,
block_count=PERMANENT_BLOCKLIST_THRESHOLD,
block_last=None)
self.session.flush()
query = self.model_query(wifis=[wifi, wifi2])
results = self.source.search(query)
self.check_model_results(results, None)
def test_not_closeby(self):
wifi = WifiShardFactory()
wifis = [
WifiShardFactory(lat=wifi.lat + 0.00001, lon=wifi.lon),
WifiShardFactory(lat=wifi.lat + 1.0, lon=wifi.lon),
WifiShardFactory(lat=wifi.lat + 1.00001, lon=wifi.lon),
]
self.session.flush()
query = self.model_query(wifis=[wifi, wifis[1]])
results = self.source.search(query)
self.check_model_results(results, None)
def test_wifi(self):
wifi = WifiShardFactory(radius=50)
wifi2 = WifiShardFactory(lat=wifi.lat,
lon=wifi.lon + 0.00001,
radius=30,
block_count=1,
block_last=None)
self.session.flush()
query = self.model_query(wifis=[wifi, wifi2])
result = self.source.search(query)
self.check_model_result(result, wifi, lon=wifi.lon + 0.000005)
def test_wifi(self, geoip_db, http_session, session, source):
wifi = WifiShardFactory(radius=5, samples=50)
wifi2 = WifiShardFactory(
lat=wifi.lat, lon=wifi.lon + 0.00001, radius=5, samples=100
)
session.flush()
query = self.model_query(geoip_db, http_session, session, wifis=[wifi, wifi2])
query.wifi[0].signalStrength = -60
query.wifi[1].signalStrength = -80
results = source.search(query)
self.check_model_results(results, [wifi], lon=wifi.lon + 0.000004)
assert results.best().score > 1.0
def test_cluster_size_over_better_signal(self):
wifi11 = WifiShardFactory()
wifi12 = WifiShardFactory(lat=wifi11.lat + 0.0002, lon=wifi11.lon)
wifi21 = WifiShardFactory(lat=wifi11.lat + 1.0, lon=wifi11.lon + 1.0)
wifi22 = WifiShardFactory(lat=wifi21.lat + 0.0002, lon=wifi21.lon)
self.session.flush()
query = self.model_query(wifis=[wifi11, wifi12, wifi21, wifi22])
query.wifi[0].signal = -100
query.wifi[1].signal = -80
query.wifi[2].signal = -100
query.wifi[3].signal = -54
result = self.source.search(query)
self.check_model_result(result, wifi21, lat=wifi21.lat + 0.0001)
def test_multiple_clusters(self):
wifi11 = WifiShardFactory()
wifi12 = WifiShardFactory(lat=wifi11.lat, lon=wifi11.lon)
wifi21 = WifiShardFactory(lat=wifi11.lat + 1.0, lon=wifi11.lon + 1.0)
wifi22 = WifiShardFactory(lat=wifi21.lat, lon=wifi21.lon)
self.session.flush()
query = self.model_query(wifis=[wifi11, wifi12, wifi21, wifi22])
query.wifi[0].signal = -100
query.wifi[1].signal = -80
query.wifi[2].signal = -100
query.wifi[3].signal = -54
results = self.source.search(query)
self.check_model_results(results, [wifi11, wifi21])
def test_wifi_temp_blocked(self):
today = util.utcnow().date()
yesterday = today - timedelta(days=1)
wifi = WifiShardFactory(radius=200)
wifi2 = WifiShardFactory(lat=wifi.lat,
lon=wifi.lon + 0.00001,
radius=300,
block_count=1,
block_last=yesterday)
self.session.flush()
query = self.model_query(wifis=[wifi, wifi2])
results = self.source.search(query)
self.check_model_results(results, None)
def test_ignore_outlier(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3,
lat=wifi.lat,
lon=wifi.lon,
radius=5)
wifis[0].lat = wifi.lat + 0.00001
wifis[1].lat = wifi.lat + 0.00002
wifis[2].lat = wifi.lat + 1.0
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis)
results = self.source.search(query)
self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001)
self.assertAlmostEqual(results.best().score, 0.15)
def test_larger_cluster_over_signal(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon)
wifis2 = WifiShardFactory.create_batch(3,
lat=wifi.lat + 1.0,
lon=wifi.lon)
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis + wifis2)
for entry in query.wifi[:-3]:
entry.signal = -80
for entry in query.wifi[-3:]:
entry.signal = -70
result = self.source.search(query)
self.check_model_result(result, wifi)
def test_larger_cluster_over_signal(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(
3, lat=wifi.lat, lon=wifi.lon)
wifis2 = WifiShardFactory.create_batch(
3, lat=wifi.lat + 1.0, lon=wifi.lon)
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis + wifis2)
for entry in query.wifi[:-3]:
entry.signal = -80
for entry in query.wifi[-3:]:
entry.signal = -70
result = self.source.search(query)
self.check_model_result(result, wifi)
def test_database_error(self, db_errors=0):
for tablename in ('cell', 'cell_area',
'ocid_cell', 'ocid_cell_area'):
self.session.execute(text('drop table %s;' % tablename))
for i in range(16):
self.session.execute(text(
'drop table wifi_shard_%s;' % hex(i)[2:]))
cells = CellFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
self.check_stats(counter=[
(self.metric_type + '.result',
['key:test', 'country:GB',
'accuracy:high', 'status:miss']),
])
self.check_raven([('ProgrammingError', db_errors)])
def test_top_results_in_noisy_cluster(self):
now = util.utcnow()
# all these should wind up in the same cluster since
# the WiFis are spaced in increments of (+1m, +1.2m)
wifi1 = WifiShardFactory.build()
wifis = []
for i in range(0, MAX_WIFIS_IN_CLUSTER + 10):
wifis.append(WifiShardFactory(lat=wifi1.lat + i * 0.00001,
lon=wifi1.lon + i * 0.000012,
samples=100 - i))
self.session.flush()
# calculate expected result
lat, lon = numpy.array(
[(wifi.lat, wifi.lon) for wifi in
wifis[:MAX_WIFIS_IN_CLUSTER]]).mean(axis=0)
score = sum([wifi.score(now) for wifi in wifis])
query = self.model_query(wifis=wifis)
for i, entry in enumerate(query.wifi):
entry.signal = -70 - i
results = self.source.search(query)
self.check_model_results(results, [wifi1], lat=lat, lon=lon)
self.assertAlmostEqual(
results.best(query.expected_accuracy).score, score, 4)
def test_weight(self, geoip_db, http_session,
session, source, stats):
wifi1 = WifiShardFactory.build()
wifis = []
for i in range(4):
wifis.append(WifiShardFactory(lat=wifi1.lat + i * 0.0001,
lon=wifi1.lon + i * 0.00012))
session.flush()
query = self.model_query(
geoip_db, http_session, session, stats,
wifis=wifis)
query.wifi[0].signalStrength = -10
query.wifi[0].age = 0
query.wifi[1].signalStrength = -40
query.wifi[1].age = 8000
query.wifi[2].signalStrength = -70
query.wifi[2].age = 16000
query.wifi[3].signalStrength = -100
results = source.search(query)
result = results.best()
assert round(result.lat, 7) == wifi1.lat + 0.0000009
assert round(result.lon, 7) == wifi1.lon + 0.0000006
assert round(result.accuracy, 2) == 39.51
def test_wifi(self):
wifis = WifiShardFactory.build_batch(2)
query = Query(wifi=[
{'macAddress': wifi.mac, 'channel': 3, 'frequency': 2412,
'signalStrength': -70, 'signalToNoiseRatio': 13,
'ssid': 'wifi'}
for wifi in wifis])
data = self._call(query.json())
assert (data == {
'wifi': [{
'bssid': wifis[0].mac,
'channel': 3,
'frequency': 2412,
'signal': -70,
'signalToNoiseRatio': 13,
}, {
'bssid': wifis[1].mac,
'channel': 3,
'frequency': 2412,
'signal': -70,
'signalToNoiseRatio': 13,
}],
'fallbacks': {'lacf': True},
'token': None,
})
def test_get_wifi(self):
wifis = WifiShardFactory.build_batch(2)
query = Query(wifi=self.wifi_model_query(wifis))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:miss']),
])
def test_update(self):
utcnow = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
# first wifi
wifi1 = WifiShardFactory(lat=None, lon=None, samples=3)
new_pos = WifiShardFactory.build()
mac1, lat1, lon1 = (wifi1.mac, new_pos.lat, new_pos.lon)
obs.extend([
obs_factory(lat=lat1,
lon=lon1, key=mac1),
obs_factory(lat=lat1 + 0.002,
lon=lon1 + 0.003, key=mac1),
obs_factory(lat=lat1 + 0.004,
lon=lon1 + 0.006, key=mac1),
])
# second wifi
wifi2 = WifiShardFactory(
lat=lat1 + 1.0, lon=lon1 + 1.0,
max_lat=lat1 + 1.0, min_lat=lat1 + 0.999,
max_lon=lon1 + 1.0, min_lon=None,
radius=20, samples=2,
created=utcnow - timedelta(10),
modified=utcnow - timedelta(10))
mac2, lat2, lon2 = (wifi2.mac, wifi2.lat, wifi2.lon)
obs.extend([
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=mac2),
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=mac2),
])
self.session.commit()
self._queue_and_update(obs)
shard = WifiShard.shard_model(mac1)
found = self.session.query(shard).filter(shard.mac == mac1).one()
self.assertAlmostEqual(found.lat, lat1 + 0.002)
self.assertAlmostEqual(found.max_lat, lat1 + 0.004)
self.assertAlmostEqual(found.min_lat, lat1)
self.assertAlmostEqual(found.lon, lon1 + 0.003)
self.assertAlmostEqual(found.max_lon, lon1 + 0.006)
self.assertAlmostEqual(found.min_lon, lon1)
self.assertEqual(found.modified.date(), utcnow.date())
self.assertEqual(found.radius, 304)
self.assertEqual(found.region, 'GB')
self.assertEqual(found.samples, 6)
shard = WifiShard.shard_model(mac2)
found = self.session.query(shard).filter(shard.mac == mac2).one()
self.assertAlmostEqual(found.lat, lat2 + 0.001)
self.assertAlmostEqual(found.max_lat, lat2 + 0.002)
self.assertAlmostEqual(found.min_lat, lat2 - 0.001)
self.assertAlmostEqual(found.lon, lon2 + 0.002)
self.assertAlmostEqual(found.max_lon, lon2 + 0.004)
self.assertAlmostEqual(found.min_lon, lon2)
self.assertEqual(found.created.date(), utcnow.date() - timedelta(10))
self.assertEqual(found.modified.date(), utcnow.date())
self.assertEqual(found.radius, 260)
self.assertEqual(found.region, 'GB')
self.assertEqual(found.samples, 4)
def test_few_candidates(self):
wifis = WifiShardFactory.create_batch(2)
self.session.flush()
query = self.model_query(wifis=[wifis[0]])
result = self.source.search(query)
self.check_model_result(result, None)
def test_top_results_in_noisy_cluster(self, geoip_db, http_session,
session, source, stats):
now = util.utcnow()
# all these should wind up in the same cluster since
# the WiFis are spaced in increments of (+0.1m, +0.12m)
wifi1 = WifiShardFactory.build()
wifis = []
for i in range(0, MAX_WIFIS_IN_CLUSTER + 10):
wifis.append(WifiShardFactory(lat=wifi1.lat + i * 0.000001,
lon=wifi1.lon + i * 0.0000012,
samples=100 - i))
session.flush()
# calculate expected result
score = sum([wifi.score(now) for wifi in wifis])
query = self.model_query(
geoip_db, http_session, session, stats,
wifis=wifis)
for i, entry in enumerate(query.wifi):
entry.signalStrength = -50 - i
results = source.search(query)
result = results.best()
assert round(result.lat, 4) == round(wifi1.lat, 4)
assert round(result.lon, 4) == round(wifi1.lon, 4)
assert round(result.score, 4) == round(score, 4)
def test_minimal(self):
wifi = WifiShardFactory.build()
data = SUBMIT_V0_SCHEMA.deserialize(
{'items': [{'lat': wifi.lat, 'lon': wifi.lon,
'wifi': [{'key': 'ab'}]}]})
assert 'items' in data
assert len(data['items']) == 1
def test_wifi(self):
wifi = WifiShardFactory.build()
self._post([{
'lat': wifi.lat,
'lon': wifi.lon,
'accuracy': 17.1,
'wifi': [{'key': wifi.mac.upper(),
'frequency': 2437,
'signal': -70,
'signalToNoiseRatio': 5,
'ssid': 'my-wifi',
}]
}])
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
self.assertEqual(item['metadata']['api_key'], None)
report = item['report']
position = report['position']
self.assertEqual(position['latitude'], wifi.lat)
self.assertEqual(position['longitude'], wifi.lon)
self.assertEqual(position['accuracy'], 17.1)
self.assertFalse('altitude' in position)
self.assertFalse('altitudeAccuracy' in position)
wifis = report['wifiAccessPoints']
self.assertEqual(wifis[0]['macAddress'], wifi.mac.upper())
self.assertFalse('channel' in wifis[0])
self.assertEqual(wifis[0]['frequency'], 2437)
self.assertEqual(wifis[0]['signalStrength'], -70)
self.assertEqual(wifis[0]['signalToNoiseRatio'], 5)
self.assertEqual(wifis[0]['ssid'], 'my-wifi')
def test_minimal(self):
wifi = WifiShardFactory.build()
data = self.schema.deserialize(
{'items': [{'lat': wifi.lat, 'lon': wifi.lon,
'wifi': [{'key': 'ab'}]}]})
self.assertTrue('items' in data)
self.assertEqual(len(data['items']), 1)
def test_check_already_good_result(self):
wifis = WifiShardFactory.build_batch(2)
internal_pos = Position(
source=DataSource.internal, lat=1.0, lon=1.0, accuracy=1.0)
query = self.model_query(wifis=wifis)
self.check_should_search(query, False, results=[internal_pos])
def test_database_error(self, db_errors=0):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, CellOCID, CellAreaOCID):
self.session.execute(text("drop table %s;" % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
self.session.execute(text("drop table %s;" % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text("drop table %s;" % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, "ok")
self.check_stats(
counter=[("request", [self.metric_path, "method:post", "status:200"])],
timer=[("request", [self.metric_path, "method:post"])],
)
if self.apikey_metrics:
self.check_stats(
counter=[
(
self.metric_type + ".result",
["key:test", "region:GB", "fallback_allowed:false", "accuracy:high", "status:miss"],
)
]
)
self.check_raven([("ProgrammingError", db_errors)])
def test_database_error(self, db_errors=0):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, CellOCID, CellAreaOCID):
self.session.execute(text('drop table %s;' % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
self.session.execute(text('drop table %s;' % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text('drop table %s;' % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
self.check_stats(counter=[
(self.metric_type + '.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
self.check_raven([('ProgrammingError', db_errors)])
def test_api_key_disallows(self):
api_key = ApiKeyFactory.build(allow_fallback=False)
cells = CellFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(query, False)
def test_wifi_miss(self):
wifis = WifiShardFactory.build_batch(2, samples=10)
self.session.flush()
query = self.model_query(wifis=wifis)
results = self.source.search(query)
self.check_model_results(results, None)
def test_bluetooth(self):
wifi = WifiShardFactory.build()
self._post(
[
{
"position": {"latitude": wifi.lat, "longitude": wifi.lon},
"bluetoothBeacons": [
{"macAddress": wifi.mac, "name": "my-beacon", "age": 3, "signalStrength": -90, "xtra_field": 4},
{"name": "beacon-2", "signalStrength": -92},
],
"wifiAccessPoints": [{"signalStrength": -52}],
}
]
)
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
report = item["report"]
self.assertTrue("timestamp" in report)
position = report["position"]
self.assertEqual(position["latitude"], wifi.lat)
self.assertEqual(position["longitude"], wifi.lon)
blues = report["bluetoothBeacons"]
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0]["macAddress"], wifi.mac)
self.assertEqual(blues[0]["age"], 3),
self.assertEqual(blues[0]["name"], "my-beacon"),
self.assertEqual(blues[0]["signalStrength"], -90),
self.assertFalse("xtra_field" in blues[0])
wifis = report["wifiAccessPoints"]
self.assertEqual(len(wifis), 0)
def test_satisfies_fail(self):
wifis = WifiShardFactory.build_batch(2)
wifi_query = [{'mac': wifi.mac} for wifi in wifis]
positions = PositionResultList(
Position(lat=1.0, lon=1.0, accuracy=2500.0, score=2.0))
query = Query(api_type='locate', wifi=wifi_query)
self.assertFalse(positions.satisfies(query))
def test_temp_blocked_admitted_again(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1)
obs = WifiObservationFactory()
WifiShardFactory(mac=obs.mac,
lat=None,
lon=None,
samples=0,
created=last_week,
modified=last_week,
block_first=last_week.date(),
block_last=last_week.date(),
block_count=1)
self.session.commit()
# add a new entry for the previously blocked wifi
self._queue_and_update([obs])
# the wifi was inserted again
shard = WifiShard.shard_model(obs.mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertEqual(wifi.block_first, last_week.date())
self.assertEqual(wifi.block_last, last_week.date())
self.assertEqual(wifi.created.date(), last_week.date())
self.assertAlmostEqual(wifi.lat, obs.lat)
self.assertAlmostEqual(wifi.lon, obs.lon)
self.assertEqual(wifi.region, 'GB')
self.assertEqual(wifi.samples, 1)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_high_hit(self):
wifis = WifiShardFactory.build_batch(2)
self._make_query(self._make_result(accuracy=1000.0), wifi=wifis)
self.check_stats(counter=[
('locate.result',
['key:key', 'country:none', 'accuracy:high', 'status:hit']),
])
def test_error(self):
wifi = WifiShardFactory.build()
self._post(
[{"position": {"latitude": wifi.lat, "longitude": wifi.lon}, "wifiAccessPoints": [{"macAddress": 10}]}],
status=400,
)
self._assert_queue_size(0)
def test_high_miss(self):
wifis = WifiShardFactory.build_batch(2)
self._make_query(self._make_result(accuracy=2500.0), wifi=wifis)
self.check_stats(counter=[
('locate.result',
['key:key', 'region:none', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
def test_wifi_duplicates(self):
wifi = WifiShardFactory.build()
query = Query(wifi=[
{'mac': wifi.mac, 'signal': -90},
{'mac': wifi.mac, 'signal': -82},
{'mac': wifi.mac, 'signal': -85},
])
self.assertEqual(len(query.wifi), 0)
def test_mixed_cell_wifi(self):
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(
cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
self.assertEqual(query.expected_accuracy, DataAccuracy.high)
def test_few_matches(self):
wifis = WifiShardFactory.create_batch(3)
wifis[0].lat = None
self.session.flush()
query = self.model_query(wifis=wifis[:2])
result = self.source.search(query)
self.check_model_result(result, None)
