def test_update_wifi(self): now = util.utcnow() k1 = 'ab1234567890' k2 = 'cd1234567890' data = [ Wifi(key=k1, new_measures=3, total_measures=3), WifiObservation(lat=1.0, lon=1.0, key=k1, created=now), WifiObservation(lat=1.002, lon=1.003, key=k1, created=now), WifiObservation(lat=1.004, lon=1.006, key=k1, created=now), Wifi(key=k2, lat=2.0, lon=2.0, new_measures=2, total_measures=2), WifiObservation(lat=2.002, lon=2.004, key=k2, created=now), WifiObservation(lat=2.002, lon=2.004, key=k2, created=now), ] self.session.add_all(data) self.session.commit() result = location_update_wifi.delay() self.assertEqual(result.get(), (2, 0)) self.check_stats( timer=['task.data.location_update_wifi'], ) wifis = dict(self.session.query(Wifi.key, Wifi).all()) self.assertEqual(set(wifis.keys()), set([k1, k2])) self.assertEqual(wifis[k1].lat, 1.002) self.assertEqual(wifis[k1].lon, 1.003) self.assertEqual(wifis[k1].new_measures, 0) self.assertEqual(wifis[k2].lat, 2.001) self.assertEqual(wifis[k2].lon, 2.002) self.assertEqual(wifis[k2].new_measures, 0)
def test_max_min_range_update(self): session = self.session k1 = "ab1234567890" k2 = "cd1234567890" data = [ Wifi(key=k1, new_measures=2, total_measures=2), WifiObservation(lat=1.0, lon=1.0, key=k1), WifiObservation(lat=1.002, lon=1.004, key=k1), Wifi(key=k2, lat=2.0, lon=-2.0, max_lat=2.001, min_lat=1.999, max_lon=-1.999, min_lon=-2.001, new_measures=2, total_measures=4), WifiObservation(lat=2.002, lon=-2.004, key=k2), WifiObservation(lat=1.998, lon=-1.996, key=k2), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) wifis = dict(session.query(Wifi.key, Wifi).all()) self.assertEqual(set(wifis.keys()), set([k1, k2])) self.assertEqual(wifis[k1].lat, 1.001) self.assertEqual(wifis[k1].max_lat, 1.002) self.assertEqual(wifis[k1].min_lat, 1.0) self.assertEqual(wifis[k1].lon, 1.002) self.assertEqual(wifis[k1].max_lon, 1.004) self.assertEqual(wifis[k1].min_lon, 1.0) self.assertEqual(wifis[k2].lat, 2.0) self.assertEqual(wifis[k2].max_lat, 2.002) self.assertEqual(wifis[k2].min_lat, 1.998) self.assertEqual(wifis[k2].lon, -2.0) self.assertEqual(wifis[k2].max_lon, -1.996) self.assertEqual(wifis[k2].min_lon, -2.004) # independent calculation: the k1 bounding box is # (1.000, 1.000) to (1.002, 1.004), with centroid # at (1.001, 1.002); worst distance from centroid # to any corner is 249m self.assertEqual(wifis[k1].range, 249) # independent calculation: the k2 bounding box is # (1.998, -2.004) to (2.002, -1.996), with centroid # at (2.000, 2.000); worst distance from centroid # to any corner is 497m self.assertEqual(wifis[k2].range, 497)
def test_max_min_range_update(self): session = self.db_master_session k1 = "ab1234567890" k2 = "cd1234567890" data = [ Wifi(key=k1, new_measures=2, total_measures=2), WifiMeasure(lat=1.0, lon=1.0, key=k1), WifiMeasure(lat=1.002, lon=1.004, key=k1), Wifi(key=k2, lat=2.0, lon=-2.0, max_lat=2.001, min_lat=1.999, max_lon=-1.999, min_lon=-2.001, new_measures=2, total_measures=4), WifiMeasure(lat=2.002, lon=-2.004, key=k2), WifiMeasure(lat=1.998, lon=-1.996, key=k2), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) wifis = dict(session.query(Wifi.key, Wifi).all()) self.assertEqual(set(wifis.keys()), set([k1, k2])) self.assertEqual(wifis[k1].lat, 1.001) self.assertEqual(wifis[k1].max_lat, 1.002) self.assertEqual(wifis[k1].min_lat, 1.0) self.assertEqual(wifis[k1].lon, 1.002) self.assertEqual(wifis[k1].max_lon, 1.004) self.assertEqual(wifis[k1].min_lon, 1.0) self.assertEqual(wifis[k2].lat, 2.0) self.assertEqual(wifis[k2].max_lat, 2.002) self.assertEqual(wifis[k2].min_lat, 1.998) self.assertEqual(wifis[k2].lon, -2.0) self.assertEqual(wifis[k2].max_lon, -1.996) self.assertEqual(wifis[k2].min_lon, -2.004) # independent calculation: the k1 bounding box is # (1.000, 1.000) to (1.002, 1.004), with centroid # at (1.001, 1.002); worst distance from centroid # to any corner is 249m self.assertEqual(wifis[k1].range, 249) # independent calculation: the k2 bounding box is # (1.998, -2.004) to (2.002, -1.996), with centroid # at (2.000, 2.000); worst distance from centroid # to any corner is 497m self.assertEqual(wifis[k2].range, 497)
def test_location_update_wifi(self): now = util.utcnow() before = now - timedelta(days=1) session = self.db_master_session k1 = "ab1234567890" k2 = "cd1234567890" data = [ Wifi(key=k1, new_measures=3, total_measures=3), WifiMeasure(lat=1.0, lon=1.0, key=k1), WifiMeasure(lat=1.002, lon=1.003, key=k1), WifiMeasure(lat=1.004, lon=1.006, key=k1), Wifi(key=k2, lat=2.0, lon=2.0, new_measures=2, total_measures=4), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old measures are skipped WifiMeasure(lat=-1.0, lon=-1.0, key=k2, created=before), WifiMeasure(lat=-1.0, lon=-1.0, key=k2, created=before), WifiMeasure(lat=2.002, lon=2.004, key=k2, created=now), WifiMeasure(lat=2.002, lon=2.004, key=k2, created=now), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) self.check_stats( total=2, timer=['task.data.location_update_wifi'], gauge=['task.data.location_update_wifi.new_measures_1_100'], ) wifis = dict(session.query(Wifi.key, Wifi).all()) self.assertEqual(set(wifis.keys()), set([k1, k2])) self.assertEqual(wifis[k1].lat, 1.002) self.assertEqual(wifis[k1].lon, 1.003) self.assertEqual(wifis[k1].new_measures, 0) self.assertEqual(wifis[k2].lat, 2.001) self.assertEqual(wifis[k2].lon, 2.002) self.assertEqual(wifis[k2].new_measures, 0)
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a wifi that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(id=month, key="ab1234567890", time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_wifi.delay( [measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_wifi.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(WifiBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(WifiMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # One wifi record should exist. self.assertEqual(session.query(Wifi).count(), 1) else: # The station existed and was seen moving, # thereby activating the blacklist. self.assertEqual(update_result.get(), (1, 1)) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(WifiBlacklist).count(), 1) self.assertEqual(session.query(Wifi).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_wifi.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_wifis(self): now = util.utcnow() long_ago = now - timedelta(days=40) session = self.db_master_session k1 = "ab1234567890" k2 = "bc1234567890" k3 = "cd1234567890" k4 = "de1234567890" k5 = "ef1234567890" k6 = "fa1234567890" keys = set([k1, k2, k3, k4, k5, k6]) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a wifi with an entry but no prior position Wifi(key=k1, new_measures=3, total_measures=0), WifiMeasure(lat=1.001, lon=1.001, key=k1), WifiMeasure(lat=1.002, lon=1.005, key=k1), WifiMeasure(lat=1.003, lon=1.009, key=k1), # a wifi with a prior known position Wifi(lat=2.0, lon=2.0, key=k2, new_measures=2, total_measures=1), WifiMeasure(lat=2.01, lon=2, key=k2), WifiMeasure(lat=2.07, lon=2, key=k2), # a wifi with a very different prior position Wifi(lat=1.0, lon=1.0, key=k3, new_measures=2, total_measures=1), WifiMeasure(lat=3.0, lon=3.0, key=k3), WifiMeasure(lat=-3.0, lon=3.0, key=k3), # another wifi with a prior known position (and negative lat) Wifi(lat=-4.0, lon=4.0, key=k4, new_measures=2, total_measures=1), WifiMeasure(lat=-4.1, lon=4, key=k4), WifiMeasure(lat=-4.16, lon=4, key=k4), # an already blacklisted wifi WifiBlacklist(key=k5), WifiMeasure(lat=5.0, lon=5.0, key=k5), WifiMeasure(lat=5.1, lon=5.0, key=k5), # a wifi with an old different record we ignore, position # estimate has been updated since Wifi(lat=6.0, lon=6.0, key=k6, new_measures=2, total_measures=1), WifiMeasure(lat=6.9, lon=6.9, key=k6, time=long_ago), WifiMeasure(lat=6.0, lon=6.0, key=k6), WifiMeasure(lat=6.001, lon=6.0, key=k6), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (5, 3)) black = session.query(WifiBlacklist).all() self.assertEqual(set([b.key for b in black]), set([k2, k3, k4, k5])) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 14) self.assertEqual(set([m.key for m in measures]), keys) # test duplicate call result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), 0) self.check_stats( total=6, timer=[ # We made duplicate calls ('task.data.location_update_wifi', 2), # One of those would've scheduled a remove_wifi task ('task.data.remove_wifi', 1) ], gauge=[ ('task.data.location_update_wifi.new_measures_1_100', 2), ])
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a wifi that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(key="ab1234567890", time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_wifi.delay([measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_wifi.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(WifiBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(WifiMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # One wifi record should exist. self.assertEqual(session.query(Wifi).count(), 1) else: # The station existed and was seen moving, # thereby activating the blacklist. self.assertEqual(update_result.get(), (1, 1)) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(WifiBlacklist).count(), 1) self.assertEqual(session.query(Wifi).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_wifi.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)