def test_should_not_provide_location_if_malformed_wifi(self):
wifi = WifiFactory.build()
malformed_wifi = WifiFactory.build()
malformed_wifi.key = 'abcd'
query = self.model_query(wifis=[wifi, malformed_wifi])
self.check_should_locate(query, False)
def test_blacklist(self):
utcnow = util.utcnow()
bad_wifi = WifiFactory.build()
good_wifi = WifiFactory.build()
black = WifiBlacklist(time=utcnow, count=1, key=bad_wifi.key)
self.session.add(black)
self.session.flush()
obs = dict(lat=good_wifi.lat, lon=good_wifi.lon)
entries = [
{'key': good_wifi.key},
{'key': good_wifi.key},
{'key': bad_wifi.key},
]
for entry in entries:
entry.update(obs)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
self.assertEqual(self.data_queue.size(), 2)
update_wifi.delay().get()
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self._compare_sets([w.key for w in wifis], [good_wifi.key])
self.check_statcounter(StatKey.wifi, 2)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_empty_wifi_entry(self):
schema = ReportV1Schema()
wifi = WifiFactory.build()
request = self._make_request(
'{"lat": %s, "lon": %s, "wifi": [{}]}' % (wifi.lat, wifi.lon))
data, errors = preprocess_request(request, schema, None)
self.assertFalse(errors)
def test_bluetooth(self):
wifi = WifiFactory.build()
self._post([{
'position': {
'latitude': wifi.lat,
'longitude': wifi.lon,
},
'bluetoothBeacons': [{
'macAddress': wifi.key,
'name': 'my-beacon',
'age': 3,
'signalStrength': -90,
'xtra_field': 4,
}],
'wifiAccessPoints': [{
'signalStrength': -52,
}]},
])
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
report = item['report']
self.assertTrue('timestamp' in report)
position = report['position']
self.assertEqual(position['latitude'], wifi.lat)
self.assertEqual(position['longitude'], wifi.lon)
blues = report['bluetoothBeacons']
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0]['macAddress'], wifi.key)
self.assertEqual(blues[0]['age'], 3),
self.assertEqual(blues[0]['name'], 'my-beacon'),
self.assertEqual(blues[0]['signalStrength'], -90),
self.assertFalse('xtra_field' in blues[0])
wifis = report['wifiAccessPoints']
self.assertEqual(len(wifis), 1)
def test_duplicated_wifi_observations(self):
session = self.session
wifi = WifiFactory.build()
self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
wifi.lat,
"longitude":
wifi.lon,
"wifiAccessPoints": [
{
"macAddress": wifi.key,
"signalStrength": -92
},
{
"macAddress": wifi.key,
"signalStrength": -77
},
]
},
]
},
status=200)
self.assertEquals(session.query(WifiObservation).count(), 1)
def test_wifi(self):
wifi = WifiFactory.build()
self._post([{
'latitude': wifi.lat,
'longitude': wifi.lon,
'wifiAccessPoints': [{
'macAddress': wifi.key,
'age': 3,
'channel': 5,
'frequency': 2437,
'radioType': '802.11n',
'signalStrength': -90,
'signalToNoiseRatio': 5,
'xtra_field': 3,
}]},
])
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
self.assertEqual(item['metadata']['api_key'], None)
report = item['report']
self.assertTrue('timestamp' in report)
position = report['position']
self.assertEqual(position['latitude'], wifi.lat)
self.assertEqual(position['longitude'], wifi.lon)
wifis = item['report']['wifiAccessPoints']
self.assertEqual(len(wifis), 1)
self.assertEqual(wifis[0]['macAddress'], wifi.key)
self.assertEqual(wifis[0]['age'], 3),
self.assertEqual(wifis[0]['channel'], 5),
self.assertEqual(wifis[0]['frequency'], 2437),
self.assertEqual(wifis[0]['radioType'], '802.11n')
self.assertEqual(wifis[0]['signalStrength'], -90),
self.assertEqual(wifis[0]['signalToNoiseRatio'], 5),
self.assertFalse('xtra_field' in wifis[0])
def test_wifi(self):
wifi = WifiFactory.build()
self._post([{
'lat': wifi.lat,
'lon': wifi.lon,
'accuracy': 17,
'wifi': [{'key': wifi.key.upper(),
'frequency': 2437,
'signal': -70,
'signalToNoiseRatio': 5,
}]
}])
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
self.assertEqual(item['metadata']['api_key'], None)
report = item['report']
position = report['position']
self.assertEqual(position['latitude'], wifi.lat)
self.assertEqual(position['longitude'], wifi.lon)
self.assertEqual(position['accuracy'], 17)
self.assertFalse('altitude' in position)
self.assertFalse('altitudeAccuracy' in position)
wifis = report['wifiAccessPoints']
self.assertEqual(wifis[0]['macAddress'], wifi.key.upper())
self.assertFalse('channel' in wifis[0])
self.assertEqual(wifis[0]['frequency'], 2437)
self.assertEqual(wifis[0]['signalStrength'], -70)
self.assertEqual(wifis[0]['signalToNoiseRatio'], 5)
def test_minimal(self):
schema = SubmitV1Schema()
wifi = WifiFactory.build()
data = schema.deserialize(
{'items': [{'lat': wifi.lat, 'lon': wifi.lon, 'wifi': [{}]}]})
self.assertTrue('items' in data)
self.assertEqual(len(data['items']), 1)
def test_error(self):
wifi = WifiFactory.build()
res = self.app.post_json(
'/v1/submit',
[{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}],
status=400)
self.assertEqual(res.json, ParseError.json_body())
self.check_raven(['ParseError'])
def test_wifi_similar_bssids_but_enough_found_clusters(self):
wifi = WifiFactory(key='00000000001f')
wifi2 = WifiFactory(key='000000000024',
lat=wifi.lat + 0.00004, lon=wifi.lon + 0.00004)
other_wifi = [
WifiFactory.build(key='000000000020'),
WifiFactory.build(key='000000000021'),
WifiFactory.build(key='000000000022'),
WifiFactory.build(key='000000000023'),
]
self.session.flush()
query = self.model_query(wifis=[wifi, wifi2] + other_wifi)
location = self.provider.locate(query)
self.check_model_location(
location, wifi,
lat=wifi.lat + 0.00002, lon=wifi.lon + 0.00002)
def test_minimal(self):
schema = SubmitV1Schema()
wifi = WifiFactory.build()
request = self._make_request(
'{"items": [{"lat": %s, "lon": %s}]}' % (wifi.lat, wifi.lon))
data, errors = preprocess_request(request, schema, None)
self.assertFalse(errors)
self.assertTrue('items' in data)
self.assertEqual(len(data['items']), 1)
def test_error(self):
wifi = WifiFactory.build()
res = self.app.post_json(
'/v1/submit',
[{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}],
status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertTrue('errors' in res.json)
self.assertFalse('status' in res.json)
self.check_raven(['JSONError'])
def test_error(self):
wifi = WifiFactory.build()
self._post([{
'latitude': wifi.lat,
'longitude': wifi.lon,
'wifiAccessPoints': [{
'macAddress': 10,
}],
}], status=400)
self._assert_queue_size(0)
def test_many_errors(self):
wifi = WifiFactory.build()
wifis = [{'wrong_key': 'ab'} for i in range(100)]
res = self.app.post_json(
'/v1/submit',
{'items': [{'lat': wifi.lat, 'lon': wifi.lon, 'wifi': wifis}]},
status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertTrue('errors' in res.json)
self.assertTrue(len(res.json['errors']) < 10)
self.check_raven(['JSONError'])
def test_error_missing_latlon(self):
wifi = WifiFactory.build()
self._post([
{'lat': wifi.lat,
'lon': wifi.lon,
'accuracy': 17,
'wifi': [{'key': wifi.key}],
},
{'wifi': [{'key': wifi.key}],
'accuracy': 16},
])
self._assert_queue_size(2)
def test_invalid_latitude(self):
session = self.session
wifi = WifiFactory.build()
self.app.post_json(
'/v1/geosubmit?key=test',
{"items": [
{"latitude": 12345.0,
"longitude": wifi.lon,
"wifiAccessPoints": [{
"macAddress": wifi.key,
}]},
]},
status=200)
self.assertEquals(session.query(WifiObservation).count(), 0)
def test_many_errors(self):
wifi = WifiFactory.build()
wifis = [{'wrong_key': 'ab'} for i in range(100)]
res = self.app.post_json(
'/v1/submit',
{'items': [{
'lat': wifi.lat,
'lon': wifi.lon,
'wifi': wifis
}]},
status=400)
self.assertEqual(res.content_type, 'application/json')
self.assertTrue('errors' in res.json)
self.assertTrue(len(res.json['errors']) < 10)
self.check_raven(['JSONError'])
def test_database_error(self):
self.session.execute(text('drop table wifi;'))
wifi = WifiFactory.build()
wifi2 = WifiFactory.build()
entries = [
{'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 7},
{'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3},
{'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3},
{'lat': wifi2.lat, 'lon': wifi2.lon, 'key': wifi2.key},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail('Unexpected exception caught: %s' % repr(exc))
self.check_raven([('ProgrammingError', 1)])
self.check_statcounter(StatKey.wifi, 0)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_invalid_latitude(self):
session = self.session
wifi = WifiFactory.build()
self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude": 12345.0,
"longitude": wifi.lon,
"wifiAccessPoints": [{
"macAddress": wifi.key,
}]
},
]
},
status=200)
self.assertEquals(session.query(WifiObservation).count(), 0)
def test_error_invalid_float(self):
wifi = WifiFactory.build()
self._post([{
'latitude': wifi.lat,
'longitude': wifi.lon,
'accuracy': float('+nan'),
'altitude': float('-inf'),
'wifiAccessPoints': [{
'macAddress': wifi.key,
}],
}])
self._assert_queue_size(1)
item = self.queue.dequeue(self.queue.queue_key())[0]
position = item['report']['position']
self.assertFalse('accuracy' in position)
self.assertFalse('altitude' in position)
def test_duplicated_wifi_observations(self):
session = self.session
wifi = WifiFactory.build()
self.app.post_json(
'/v1/geosubmit?key=test',
{"items": [
{"latitude": wifi.lat,
"longitude": wifi.lon,
"wifiAccessPoints": [
{"macAddress": wifi.key,
"signalStrength": -92},
{"macAddress": wifi.key,
"signalStrength": -77},
]},
]},
status=200)
self.assertEquals(session.query(WifiObservation).count(), 1)
def test_insert_observations(self):
session = self.session
time = util.utcnow() - timedelta(days=1)
wifi = WifiFactory(total_measures=0)
wifi2 = WifiFactory.build(total_measures=0)
user = User(nickname=u'test')
session.add(user)
session.flush()
obs = dict(
created=time, time=time, lat=wifi.lat, lon=wifi.lon,
accuracy=None, altitude=None, altitude_accuracy=None,
heading=52.9, speed=158.5,
)
entries = [
{'key': wifi.key, 'channel': 11, 'signal': -80},
{'key': wifi.key, 'channel': 3, 'signal': -90},
{'key': wifi.key, 'channel': 3, 'signal': -80},
{'key': wifi2.key, 'channel': 3, 'signal': -90},
]
for entry in entries:
entry.update(obs)
result = insert_measures_wifi.delay(entries, userid=user.id)
self.assertEqual(result.get(), 4)
self.assertEqual(self.data_queue.size(), 4)
update_wifi.delay().get()
self.session.refresh(wifi)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self._compare_sets([w.key for w in wifis], [wifi.key, wifi2.key])
self._compare_sets([w.total_measures for w in wifis], [1, 3])
score_queue = self.celery_app.data_queues['update_score']
scores = score_queue.dequeue()
self.assertEqual(len(scores), 1)
score = scores[0]
self.assertEqual(score['hashkey'].userid, user.id)
self.assertEqual(score['hashkey'].key, ScoreKey.new_wifi)
self.assertEqual(score['value'], 1)
self.check_statcounter(StatKey.wifi, 4)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_invalid_float(self):
wifi = WifiFactory.build()
self.app.post_json(
'/v1/geosubmit?key=test',
{"items": [
{"latitude": wifi.lat,
"longitude": wifi.lon,
"accuracy": float('+nan'),
"altitude": float('-inf'),
"wifiAccessPoints": [{
"macAddress": wifi.key,
}]},
]},
status=200)
obs = self.session.query(WifiObservation).all()
self.assertEqual(len(obs), 1)
self.assertFalse(obs[0].accuracy)
self.assertFalse(obs[0].altitude)
def test_ok_no_existing_wifi(self):
session = self.session
wifi = WifiFactory.build()
res = self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude":
wifi.lat,
"longitude":
wifi.lon,
"wifiAccessPoints": [{
"macAddress": wifi.key,
"age": 3,
"channel": 6,
"frequency": 2437,
"signalToNoiseRatio": 13,
"signalStrength": -77,
}]
},
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {})
# Check that wifi exists
query = session.query(Wifi)
count = query.filter(Wifi.key == wifi.key).count()
self.assertEquals(count, 1)
# check that WifiObservation records are created
result = session.query(WifiObservation).all()
self.assertEquals(len(result), 1)
obs = result[0]
self.assertEqual(obs.lat, wifi.lat)
self.assertEqual(obs.lon, wifi.lon)
self.assertEqual(obs.key, wifi.key)
self.assertEqual(obs.channel, 6)
self.assertEqual(obs.signal, -77)
self.assertEqual(obs.snr, 13)
def test_invalid_float(self):
wifi = WifiFactory.build()
self.app.post_json('/v1/geosubmit?key=test', {
"items": [
{
"latitude": wifi.lat,
"longitude": wifi.lon,
"accuracy": float('+nan'),
"altitude": float('-inf'),
"wifiAccessPoints": [{
"macAddress": wifi.key,
}]
},
]
},
status=200)
obs = self.session.query(WifiObservation).all()
self.assertEqual(len(obs), 1)
self.assertFalse(obs[0].accuracy)
self.assertFalse(obs[0].altitude)
def test_wifi_only_use_top_five_signals_in_noisy_cluster(self):
# all these should wind up in the same cluster since
# clustering threshold is 500m and the 10 wifis are
# spaced in increments of (+1m, +1.2m)
wifi = WifiFactory.build()
wifis = []
for i in range(0, 10):
wifis.append(WifiFactory(lat=wifi.lat + i * 0.00001,
lon=wifi.lon + i * 0.000012))
self.session.flush()
query = self.model_query(wifis=wifis)
for i, entry in enumerate(query.wifi):
entry['signal'] = -70 - i
location = self.provider.locate(query)
self.check_model_location(
location, wifi,
lat=wifi.lat + 0.00002,
lon=wifi.lon + 0.000024)
def test_update_wifi(self):
obs = []
obs_factory = WifiObservationFactory
# first wifi
wifi1 = WifiFactory(lat=None, lon=None, total_measures=3)
new_pos = WifiFactory.build()
lat1, lon1 = (new_pos.lat, new_pos.lon)
obs.extend([
obs_factory(lat=lat1,
lon=lon1, key=wifi1.key),
obs_factory(lat=lat1 + 0.002,
lon=lon1 + 0.003, key=wifi1.key),
obs_factory(lat=lat1 + 0.004,
lon=lon1 + 0.006, key=wifi1.key),
])
# second wifi
wifi2 = WifiFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, total_measures=2)
lat2, lon2 = (wifi2.lat, wifi2.lon)
obs.extend([
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=wifi2.key),
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=wifi2.key),
])
self.data_queue.enqueue(obs)
self.session.flush()
result = update_wifi.delay()
self.assertEqual(result.get(), (2, 0))
self.check_stats(
timer=['task.data.update_wifi'],
)
self.session.refresh(wifi1)
self.session.refresh(wifi2)
found = dict(self.session.query(Wifi.key, Wifi).all())
self.assertEqual(set(found.keys()), set([wifi1.key, wifi2.key]))
self.assertAlmostEqual(found[wifi1.key].lat, lat1 + 0.002)
self.assertAlmostEqual(found[wifi1.key].lon, lon1 + 0.003)
self.assertAlmostEqual(found[wifi2.key].lat, lat2 + 0.001)
self.assertAlmostEqual(found[wifi2.key].lon, lon2 + 0.002)
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
wifi = WifiFactory.build()
self.session.add(WifiBlacklist(time=last_week, count=1, key=wifi.key))
self.session.flush()
# add a new entry for the previously blacklisted wifi
obs = dict(lat=wifi.lat, lon=wifi.lon, key=wifi.key)
insert_measures_wifi.delay([obs]).get()
self.assertEqual(self.data_queue.size(), 1)
update_wifi.delay().get()
# the wifi was inserted again
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(wifis[0].created, last_week)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_ok_no_existing_wifi(self):
session = self.session
wifi = WifiFactory.build()
res = self.app.post_json(
'/v1/geosubmit?key=test',
{"items": [
{"latitude": wifi.lat,
"longitude": wifi.lon,
"wifiAccessPoints": [{
"macAddress": wifi.key,
"age": 3,
"channel": 6,
"frequency": 2437,
"signalToNoiseRatio": 13,
"signalStrength": -77,
}]},
]},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {})
# Check that wifi exists
query = session.query(Wifi)
count = query.filter(Wifi.key == wifi.key).count()
self.assertEquals(count, 1)
# check that WifiObservation records are created
result = session.query(WifiObservation).all()
self.assertEquals(len(result), 1)
obs = result[0]
self.assertEqual(obs.lat, wifi.lat)
self.assertEqual(obs.lon, wifi.lon)
self.assertEqual(obs.key, wifi.key)
self.assertEqual(obs.channel, 6)
self.assertEqual(obs.signal, -77)
self.assertEqual(obs.snr, 13)
def test_wifi_malformed(self):
wifi = WifiFactory.build()
wifi_query = {'key': wifi.key}
query = Query(wifi=[wifi_query, {'key': 'foo'}])
self.assertEqual(len(query.wifi), 0)
def test_wifi_single(self):
wifi = WifiFactory.build()
wifi_query = {'key': wifi.key}
query = Query(wifi=[wifi_query])
self.assertEqual(len(query.wifi), 0)
def test_blacklist_moving_wifis(self):
now = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
moving = set()
wifis = WifiFactory.create_batch(4)
wifis.append(WifiFactory.build())
# a wifi with an entry but no prior position
wifi = wifis[0]
wifi.total_measures = 0
obs.extend([
obs_factory(lat=wifi.lat + 0.001,
lon=wifi.lon + 0.001, key=wifi.key),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.005, key=wifi.key),
obs_factory(lat=wifi.lat + 0.003,
lon=wifi.lon + 0.009, key=wifi.key),
])
wifi.lat = None
wifi.lon = None
# a wifi with a prior known position
wifi = wifis[1]
wifi.total_measures = 1
wifi.lat += 1.0
wifi.lon += 1.0
obs.extend([
obs_factory(lat=wifi.lat + 0.01,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat + 0.07,
lon=wifi.lon, key=wifi.key),
])
moving.add(wifi.hashkey())
# a wifi with a very different prior position
wifi = wifis[2]
wifi.total_measures = 1
obs.extend([
obs_factory(lat=wifi.lat + 2.0,
lon=wifi.lon + 2.0, key=wifi.key),
obs_factory(lat=wifi.lat - 4.0,
lon=wifi.lon + 2.0, key=wifi.key),
])
moving.add(wifi.hashkey())
# another wifi with a prior known position (and negative lat)
wifi = wifis[3]
wifi.total_measures = 1
wifi.lat *= -1.0
obs.extend([
obs_factory(lat=wifi.lat - 0.1,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat - 0.16,
lon=wifi.lon, key=wifi.key),
])
moving.add(wifi.hashkey())
# an already blacklisted wifi
wifi = wifis[4]
WifiBlacklistFactory(key=wifi.key, time=now, count=1)
obs.extend([
obs_factory(lat=wifi.lat,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat + 0.1,
lon=wifi.lon, key=wifi.key),
])
moving.add(wifi.hashkey())
self.data_queue.enqueue(obs)
self.session.commit()
result = update_wifi.delay()
self.assertEqual(result.get(), (4, 3))
black = self.session.query(WifiBlacklist).all()
self.assertEqual(set([b.hashkey() for b in black]), moving)
# test duplicate call
result = update_wifi.delay()
self.assertEqual(result.get(), (0, 0))
self.check_stats(
timer=[
# We made duplicate calls
('task.data.update_wifi', 2),
# One of those would've scheduled a remove_wifi task
('task.data.remove_wifi', 1)
])
def test_should_not_provide_location_if_one_wifi_provided(self):
wifi = WifiFactory.build()
query = self.model_query(wifis=[wifi])
self.check_should_locate(query, False)
def test_empty_wifi_entry(self):
schema = ReportV1Schema()
wifi = WifiFactory.build()
schema.deserialize({'lat': wifi.lat, 'lon': wifi.lon, 'wifi': [{}]})
