def test_should_not_provide_location_if_malformed_wifi(self): wifi = WifiFactory.build() malformed_wifi = WifiFactory.build() malformed_wifi.key = 'abcd' query = self.model_query(wifis=[wifi, malformed_wifi]) self.check_should_locate(query, False)
def test_blacklist(self): utcnow = util.utcnow() bad_wifi = WifiFactory.build() good_wifi = WifiFactory.build() black = WifiBlacklist(time=utcnow, count=1, key=bad_wifi.key) self.session.add(black) self.session.flush() obs = dict(lat=good_wifi.lat, lon=good_wifi.lon) entries = [ {'key': good_wifi.key}, {'key': good_wifi.key}, {'key': bad_wifi.key}, ] for entry in entries: entry.update(obs) result = insert_measures_wifi.delay(entries) self.assertEqual(result.get(), 2) self.assertEqual(self.data_queue.size(), 2) update_wifi.delay().get() wifis = self.session.query(Wifi).all() self.assertEqual(len(wifis), 1) self._compare_sets([w.key for w in wifis], [good_wifi.key]) self.check_statcounter(StatKey.wifi, 2) self.check_statcounter(StatKey.unique_wifi, 1)
def test_empty_wifi_entry(self): schema = ReportV1Schema() wifi = WifiFactory.build() request = self._make_request( '{"lat": %s, "lon": %s, "wifi": [{}]}' % (wifi.lat, wifi.lon)) data, errors = preprocess_request(request, schema, None) self.assertFalse(errors)
def test_bluetooth(self): wifi = WifiFactory.build() self._post([{ 'position': { 'latitude': wifi.lat, 'longitude': wifi.lon, }, 'bluetoothBeacons': [{ 'macAddress': wifi.key, 'name': 'my-beacon', 'age': 3, 'signalStrength': -90, 'xtra_field': 4, }], 'wifiAccessPoints': [{ 'signalStrength': -52, }]}, ]) self._assert_queue_size(1) item = self.queue.dequeue(self.queue.queue_key())[0] report = item['report'] self.assertTrue('timestamp' in report) position = report['position'] self.assertEqual(position['latitude'], wifi.lat) self.assertEqual(position['longitude'], wifi.lon) blues = report['bluetoothBeacons'] self.assertEqual(len(blues), 1) self.assertEqual(blues[0]['macAddress'], wifi.key) self.assertEqual(blues[0]['age'], 3), self.assertEqual(blues[0]['name'], 'my-beacon'), self.assertEqual(blues[0]['signalStrength'], -90), self.assertFalse('xtra_field' in blues[0]) wifis = report['wifiAccessPoints'] self.assertEqual(len(wifis), 1)
def test_duplicated_wifi_observations(self): session = self.session wifi = WifiFactory.build() self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": wifi.lat, "longitude": wifi.lon, "wifiAccessPoints": [ { "macAddress": wifi.key, "signalStrength": -92 }, { "macAddress": wifi.key, "signalStrength": -77 }, ] }, ] }, status=200) self.assertEquals(session.query(WifiObservation).count(), 1)
def test_wifi(self): wifi = WifiFactory.build() self._post([{ 'latitude': wifi.lat, 'longitude': wifi.lon, 'wifiAccessPoints': [{ 'macAddress': wifi.key, 'age': 3, 'channel': 5, 'frequency': 2437, 'radioType': '802.11n', 'signalStrength': -90, 'signalToNoiseRatio': 5, 'xtra_field': 3, }]}, ]) self._assert_queue_size(1) item = self.queue.dequeue(self.queue.queue_key())[0] self.assertEqual(item['metadata']['api_key'], None) report = item['report'] self.assertTrue('timestamp' in report) position = report['position'] self.assertEqual(position['latitude'], wifi.lat) self.assertEqual(position['longitude'], wifi.lon) wifis = item['report']['wifiAccessPoints'] self.assertEqual(len(wifis), 1) self.assertEqual(wifis[0]['macAddress'], wifi.key) self.assertEqual(wifis[0]['age'], 3), self.assertEqual(wifis[0]['channel'], 5), self.assertEqual(wifis[0]['frequency'], 2437), self.assertEqual(wifis[0]['radioType'], '802.11n') self.assertEqual(wifis[0]['signalStrength'], -90), self.assertEqual(wifis[0]['signalToNoiseRatio'], 5), self.assertFalse('xtra_field' in wifis[0])
def test_wifi(self): wifi = WifiFactory.build() self._post([{ 'lat': wifi.lat, 'lon': wifi.lon, 'accuracy': 17, 'wifi': [{'key': wifi.key.upper(), 'frequency': 2437, 'signal': -70, 'signalToNoiseRatio': 5, }] }]) self._assert_queue_size(1) item = self.queue.dequeue(self.queue.queue_key())[0] self.assertEqual(item['metadata']['api_key'], None) report = item['report'] position = report['position'] self.assertEqual(position['latitude'], wifi.lat) self.assertEqual(position['longitude'], wifi.lon) self.assertEqual(position['accuracy'], 17) self.assertFalse('altitude' in position) self.assertFalse('altitudeAccuracy' in position) wifis = report['wifiAccessPoints'] self.assertEqual(wifis[0]['macAddress'], wifi.key.upper()) self.assertFalse('channel' in wifis[0]) self.assertEqual(wifis[0]['frequency'], 2437) self.assertEqual(wifis[0]['signalStrength'], -70) self.assertEqual(wifis[0]['signalToNoiseRatio'], 5)
def test_minimal(self): schema = SubmitV1Schema() wifi = WifiFactory.build() data = schema.deserialize( {'items': [{'lat': wifi.lat, 'lon': wifi.lon, 'wifi': [{}]}]}) self.assertTrue('items' in data) self.assertEqual(len(data['items']), 1)
def test_error(self): wifi = WifiFactory.build() res = self.app.post_json( '/v1/submit', [{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}], status=400) self.assertEqual(res.json, ParseError.json_body()) self.check_raven(['ParseError'])
def test_wifi_similar_bssids_but_enough_found_clusters(self): wifi = WifiFactory(key='00000000001f') wifi2 = WifiFactory(key='000000000024', lat=wifi.lat + 0.00004, lon=wifi.lon + 0.00004) other_wifi = [ WifiFactory.build(key='000000000020'), WifiFactory.build(key='000000000021'), WifiFactory.build(key='000000000022'), WifiFactory.build(key='000000000023'), ] self.session.flush() query = self.model_query(wifis=[wifi, wifi2] + other_wifi) location = self.provider.locate(query) self.check_model_location( location, wifi, lat=wifi.lat + 0.00002, lon=wifi.lon + 0.00002)
def test_minimal(self): schema = SubmitV1Schema() wifi = WifiFactory.build() request = self._make_request( '{"items": [{"lat": %s, "lon": %s}]}' % (wifi.lat, wifi.lon)) data, errors = preprocess_request(request, schema, None) self.assertFalse(errors) self.assertTrue('items' in data) self.assertEqual(len(data['items']), 1)
def test_error(self): wifi = WifiFactory.build() res = self.app.post_json( '/v1/submit', [{'lat': wifi.lat, 'lon': wifi.lon, 'cell': []}], status=400) self.assertEqual(res.content_type, 'application/json') self.assertTrue('errors' in res.json) self.assertFalse('status' in res.json) self.check_raven(['JSONError'])
def test_error(self): wifi = WifiFactory.build() self._post([{ 'latitude': wifi.lat, 'longitude': wifi.lon, 'wifiAccessPoints': [{ 'macAddress': 10, }], }], status=400) self._assert_queue_size(0)
def test_many_errors(self): wifi = WifiFactory.build() wifis = [{'wrong_key': 'ab'} for i in range(100)] res = self.app.post_json( '/v1/submit', {'items': [{'lat': wifi.lat, 'lon': wifi.lon, 'wifi': wifis}]}, status=400) self.assertEqual(res.content_type, 'application/json') self.assertTrue('errors' in res.json) self.assertTrue(len(res.json['errors']) < 10) self.check_raven(['JSONError'])
def test_error_missing_latlon(self): wifi = WifiFactory.build() self._post([ {'lat': wifi.lat, 'lon': wifi.lon, 'accuracy': 17, 'wifi': [{'key': wifi.key}], }, {'wifi': [{'key': wifi.key}], 'accuracy': 16}, ]) self._assert_queue_size(2)
def test_invalid_latitude(self): session = self.session wifi = WifiFactory.build() self.app.post_json( '/v1/geosubmit?key=test', {"items": [ {"latitude": 12345.0, "longitude": wifi.lon, "wifiAccessPoints": [{ "macAddress": wifi.key, }]}, ]}, status=200) self.assertEquals(session.query(WifiObservation).count(), 0)
def test_many_errors(self): wifi = WifiFactory.build() wifis = [{'wrong_key': 'ab'} for i in range(100)] res = self.app.post_json( '/v1/submit', {'items': [{ 'lat': wifi.lat, 'lon': wifi.lon, 'wifi': wifis }]}, status=400) self.assertEqual(res.content_type, 'application/json') self.assertTrue('errors' in res.json) self.assertTrue(len(res.json['errors']) < 10) self.check_raven(['JSONError'])
def test_database_error(self): self.session.execute(text('drop table wifi;')) wifi = WifiFactory.build() wifi2 = WifiFactory.build() entries = [ {'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 7}, {'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3}, {'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3}, {'lat': wifi2.lat, 'lon': wifi2.lon, 'key': wifi2.key}, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail('Unexpected exception caught: %s' % repr(exc)) self.check_raven([('ProgrammingError', 1)]) self.check_statcounter(StatKey.wifi, 0) self.check_statcounter(StatKey.unique_wifi, 0)
def test_invalid_latitude(self): session = self.session wifi = WifiFactory.build() self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": 12345.0, "longitude": wifi.lon, "wifiAccessPoints": [{ "macAddress": wifi.key, }] }, ] }, status=200) self.assertEquals(session.query(WifiObservation).count(), 0)
def test_error_invalid_float(self): wifi = WifiFactory.build() self._post([{ 'latitude': wifi.lat, 'longitude': wifi.lon, 'accuracy': float('+nan'), 'altitude': float('-inf'), 'wifiAccessPoints': [{ 'macAddress': wifi.key, }], }]) self._assert_queue_size(1) item = self.queue.dequeue(self.queue.queue_key())[0] position = item['report']['position'] self.assertFalse('accuracy' in position) self.assertFalse('altitude' in position)
def test_duplicated_wifi_observations(self): session = self.session wifi = WifiFactory.build() self.app.post_json( '/v1/geosubmit?key=test', {"items": [ {"latitude": wifi.lat, "longitude": wifi.lon, "wifiAccessPoints": [ {"macAddress": wifi.key, "signalStrength": -92}, {"macAddress": wifi.key, "signalStrength": -77}, ]}, ]}, status=200) self.assertEquals(session.query(WifiObservation).count(), 1)
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) wifi = WifiFactory(total_measures=0) wifi2 = WifiFactory.build(total_measures=0) user = User(nickname=u'test') session.add(user) session.flush() obs = dict( created=time, time=time, lat=wifi.lat, lon=wifi.lon, accuracy=None, altitude=None, altitude_accuracy=None, heading=52.9, speed=158.5, ) entries = [ {'key': wifi.key, 'channel': 11, 'signal': -80}, {'key': wifi.key, 'channel': 3, 'signal': -90}, {'key': wifi.key, 'channel': 3, 'signal': -80}, {'key': wifi2.key, 'channel': 3, 'signal': -90}, ] for entry in entries: entry.update(obs) result = insert_measures_wifi.delay(entries, userid=user.id) self.assertEqual(result.get(), 4) self.assertEqual(self.data_queue.size(), 4) update_wifi.delay().get() self.session.refresh(wifi) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self._compare_sets([w.key for w in wifis], [wifi.key, wifi2.key]) self._compare_sets([w.total_measures for w in wifis], [1, 3]) score_queue = self.celery_app.data_queues['update_score'] scores = score_queue.dequeue() self.assertEqual(len(scores), 1) score = scores[0] self.assertEqual(score['hashkey'].userid, user.id) self.assertEqual(score['hashkey'].key, ScoreKey.new_wifi) self.assertEqual(score['value'], 1) self.check_statcounter(StatKey.wifi, 4) self.check_statcounter(StatKey.unique_wifi, 1)
def test_invalid_float(self): wifi = WifiFactory.build() self.app.post_json( '/v1/geosubmit?key=test', {"items": [ {"latitude": wifi.lat, "longitude": wifi.lon, "accuracy": float('+nan'), "altitude": float('-inf'), "wifiAccessPoints": [{ "macAddress": wifi.key, }]}, ]}, status=200) obs = self.session.query(WifiObservation).all() self.assertEqual(len(obs), 1) self.assertFalse(obs[0].accuracy) self.assertFalse(obs[0].altitude)
def test_ok_no_existing_wifi(self): session = self.session wifi = WifiFactory.build() res = self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": wifi.lat, "longitude": wifi.lon, "wifiAccessPoints": [{ "macAddress": wifi.key, "age": 3, "channel": 6, "frequency": 2437, "signalToNoiseRatio": 13, "signalStrength": -77, }] }, ] }, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {}) # Check that wifi exists query = session.query(Wifi) count = query.filter(Wifi.key == wifi.key).count() self.assertEquals(count, 1) # check that WifiObservation records are created result = session.query(WifiObservation).all() self.assertEquals(len(result), 1) obs = result[0] self.assertEqual(obs.lat, wifi.lat) self.assertEqual(obs.lon, wifi.lon) self.assertEqual(obs.key, wifi.key) self.assertEqual(obs.channel, 6) self.assertEqual(obs.signal, -77) self.assertEqual(obs.snr, 13)
def test_invalid_float(self): wifi = WifiFactory.build() self.app.post_json('/v1/geosubmit?key=test', { "items": [ { "latitude": wifi.lat, "longitude": wifi.lon, "accuracy": float('+nan'), "altitude": float('-inf'), "wifiAccessPoints": [{ "macAddress": wifi.key, }] }, ] }, status=200) obs = self.session.query(WifiObservation).all() self.assertEqual(len(obs), 1) self.assertFalse(obs[0].accuracy) self.assertFalse(obs[0].altitude)
def test_wifi_only_use_top_five_signals_in_noisy_cluster(self): # all these should wind up in the same cluster since # clustering threshold is 500m and the 10 wifis are # spaced in increments of (+1m, +1.2m) wifi = WifiFactory.build() wifis = [] for i in range(0, 10): wifis.append(WifiFactory(lat=wifi.lat + i * 0.00001, lon=wifi.lon + i * 0.000012)) self.session.flush() query = self.model_query(wifis=wifis) for i, entry in enumerate(query.wifi): entry['signal'] = -70 - i location = self.provider.locate(query) self.check_model_location( location, wifi, lat=wifi.lat + 0.00002, lon=wifi.lon + 0.000024)
def test_update_wifi(self): obs = [] obs_factory = WifiObservationFactory # first wifi wifi1 = WifiFactory(lat=None, lon=None, total_measures=3) new_pos = WifiFactory.build() lat1, lon1 = (new_pos.lat, new_pos.lon) obs.extend([ obs_factory(lat=lat1, lon=lon1, key=wifi1.key), obs_factory(lat=lat1 + 0.002, lon=lon1 + 0.003, key=wifi1.key), obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, key=wifi1.key), ]) # second wifi wifi2 = WifiFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, total_measures=2) lat2, lon2 = (wifi2.lat, wifi2.lon) obs.extend([ obs_factory(lat=lat2 + 0.002, lon=lon2 + 0.004, key=wifi2.key), obs_factory(lat=lat2 + 0.002, lon=lon2 + 0.004, key=wifi2.key), ]) self.data_queue.enqueue(obs) self.session.flush() result = update_wifi.delay() self.assertEqual(result.get(), (2, 0)) self.check_stats( timer=['task.data.update_wifi'], ) self.session.refresh(wifi1) self.session.refresh(wifi2) found = dict(self.session.query(Wifi.key, Wifi).all()) self.assertEqual(set(found.keys()), set([wifi1.key, wifi2.key])) self.assertAlmostEqual(found[wifi1.key].lat, lat1 + 0.002) self.assertAlmostEqual(found[wifi1.key].lon, lon1 + 0.003) self.assertAlmostEqual(found[wifi2.key].lat, lat2 + 0.001) self.assertAlmostEqual(found[wifi2.key].lon, lon2 + 0.002)
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) wifi = WifiFactory.build() self.session.add(WifiBlacklist(time=last_week, count=1, key=wifi.key)) self.session.flush() # add a new entry for the previously blacklisted wifi obs = dict(lat=wifi.lat, lon=wifi.lon, key=wifi.key) insert_measures_wifi.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_wifi.delay().get() # the wifi was inserted again wifis = self.session.query(Wifi).all() self.assertEqual(len(wifis), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(wifis[0].created, last_week) self.check_statcounter(StatKey.unique_wifi, 0)
def test_ok_no_existing_wifi(self): session = self.session wifi = WifiFactory.build() res = self.app.post_json( '/v1/geosubmit?key=test', {"items": [ {"latitude": wifi.lat, "longitude": wifi.lon, "wifiAccessPoints": [{ "macAddress": wifi.key, "age": 3, "channel": 6, "frequency": 2437, "signalToNoiseRatio": 13, "signalStrength": -77, }]}, ]}, status=200) self.assertEqual(res.content_type, 'application/json') self.assertEqual(res.json, {}) # Check that wifi exists query = session.query(Wifi) count = query.filter(Wifi.key == wifi.key).count() self.assertEquals(count, 1) # check that WifiObservation records are created result = session.query(WifiObservation).all() self.assertEquals(len(result), 1) obs = result[0] self.assertEqual(obs.lat, wifi.lat) self.assertEqual(obs.lon, wifi.lon) self.assertEqual(obs.key, wifi.key) self.assertEqual(obs.channel, 6) self.assertEqual(obs.signal, -77) self.assertEqual(obs.snr, 13)
def test_wifi_malformed(self): wifi = WifiFactory.build() wifi_query = {'key': wifi.key} query = Query(wifi=[wifi_query, {'key': 'foo'}]) self.assertEqual(len(query.wifi), 0)
def test_wifi_single(self): wifi = WifiFactory.build() wifi_query = {'key': wifi.key} query = Query(wifi=[wifi_query]) self.assertEqual(len(query.wifi), 0)
def test_blacklist_moving_wifis(self): now = util.utcnow() obs = [] obs_factory = WifiObservationFactory moving = set() wifis = WifiFactory.create_batch(4) wifis.append(WifiFactory.build()) # a wifi with an entry but no prior position wifi = wifis[0] wifi.total_measures = 0 obs.extend([ obs_factory(lat=wifi.lat + 0.001, lon=wifi.lon + 0.001, key=wifi.key), obs_factory(lat=wifi.lat + 0.002, lon=wifi.lon + 0.005, key=wifi.key), obs_factory(lat=wifi.lat + 0.003, lon=wifi.lon + 0.009, key=wifi.key), ]) wifi.lat = None wifi.lon = None # a wifi with a prior known position wifi = wifis[1] wifi.total_measures = 1 wifi.lat += 1.0 wifi.lon += 1.0 obs.extend([ obs_factory(lat=wifi.lat + 0.01, lon=wifi.lon, key=wifi.key), obs_factory(lat=wifi.lat + 0.07, lon=wifi.lon, key=wifi.key), ]) moving.add(wifi.hashkey()) # a wifi with a very different prior position wifi = wifis[2] wifi.total_measures = 1 obs.extend([ obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon + 2.0, key=wifi.key), obs_factory(lat=wifi.lat - 4.0, lon=wifi.lon + 2.0, key=wifi.key), ]) moving.add(wifi.hashkey()) # another wifi with a prior known position (and negative lat) wifi = wifis[3] wifi.total_measures = 1 wifi.lat *= -1.0 obs.extend([ obs_factory(lat=wifi.lat - 0.1, lon=wifi.lon, key=wifi.key), obs_factory(lat=wifi.lat - 0.16, lon=wifi.lon, key=wifi.key), ]) moving.add(wifi.hashkey()) # an already blacklisted wifi wifi = wifis[4] WifiBlacklistFactory(key=wifi.key, time=now, count=1) obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.key), obs_factory(lat=wifi.lat + 0.1, lon=wifi.lon, key=wifi.key), ]) moving.add(wifi.hashkey()) self.data_queue.enqueue(obs) self.session.commit() result = update_wifi.delay() self.assertEqual(result.get(), (4, 3)) black = self.session.query(WifiBlacklist).all() self.assertEqual(set([b.hashkey() for b in black]), moving) # test duplicate call result = update_wifi.delay() self.assertEqual(result.get(), (0, 0)) self.check_stats( timer=[ # We made duplicate calls ('task.data.update_wifi', 2), # One of those would've scheduled a remove_wifi task ('task.data.remove_wifi', 1) ])
def test_should_not_provide_location_if_one_wifi_provided(self): wifi = WifiFactory.build() query = self.model_query(wifis=[wifi]) self.check_should_locate(query, False)
def test_empty_wifi_entry(self): schema = ReportV1Schema() wifi = WifiFactory.build() schema.deserialize({'lat': wifi.lat, 'lon': wifi.lon, 'wifi': [{}]})