def test_update_wifi(self):
now = util.utcnow()
k1 = 'ab1234567890'
k2 = 'cd1234567890'
data = [
Wifi(key=k1, new_measures=3, total_measures=3),
WifiObservation(lat=1.0, lon=1.0, key=k1, created=now),
WifiObservation(lat=1.002, lon=1.003, key=k1, created=now),
WifiObservation(lat=1.004, lon=1.006, key=k1, created=now),
Wifi(key=k2, lat=2.0, lon=2.0,
new_measures=2, total_measures=2),
WifiObservation(lat=2.002, lon=2.004, key=k2, created=now),
WifiObservation(lat=2.002, lon=2.004, key=k2, created=now),
]
self.session.add_all(data)
self.session.commit()
result = location_update_wifi.delay()
self.assertEqual(result.get(), (2, 0))
self.check_stats(
timer=['task.data.location_update_wifi'],
)
wifis = dict(self.session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.002)
self.assertEqual(wifis[k1].lon, 1.003)
self.assertEqual(wifis[k1].new_measures, 0)
self.assertEqual(wifis[k2].lat, 2.001)
self.assertEqual(wifis[k2].lon, 2.002)
self.assertEqual(wifis[k2].new_measures, 0)
def test_max_min_range_update(self):
session = self.session
k1 = "ab1234567890"
k2 = "cd1234567890"
data = [
Wifi(key=k1, new_measures=2, total_measures=2),
WifiObservation(lat=1.0, lon=1.0, key=k1),
WifiObservation(lat=1.002, lon=1.004, key=k1),
Wifi(key=k2,
lat=2.0,
lon=-2.0,
max_lat=2.001,
min_lat=1.999,
max_lon=-1.999,
min_lon=-2.001,
new_measures=2,
total_measures=4),
WifiObservation(lat=2.002, lon=-2.004, key=k2),
WifiObservation(lat=1.998, lon=-1.996, key=k2),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
wifis = dict(session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.001)
self.assertEqual(wifis[k1].max_lat, 1.002)
self.assertEqual(wifis[k1].min_lat, 1.0)
self.assertEqual(wifis[k1].lon, 1.002)
self.assertEqual(wifis[k1].max_lon, 1.004)
self.assertEqual(wifis[k1].min_lon, 1.0)
self.assertEqual(wifis[k2].lat, 2.0)
self.assertEqual(wifis[k2].max_lat, 2.002)
self.assertEqual(wifis[k2].min_lat, 1.998)
self.assertEqual(wifis[k2].lon, -2.0)
self.assertEqual(wifis[k2].max_lon, -1.996)
self.assertEqual(wifis[k2].min_lon, -2.004)
# independent calculation: the k1 bounding box is
# (1.000, 1.000) to (1.002, 1.004), with centroid
# at (1.001, 1.002); worst distance from centroid
# to any corner is 249m
self.assertEqual(wifis[k1].range, 249)
# independent calculation: the k2 bounding box is
# (1.998, -2.004) to (2.002, -1.996), with centroid
# at (2.000, 2.000); worst distance from centroid
# to any corner is 497m
self.assertEqual(wifis[k2].range, 497)
def test_max_min_range_update(self):
session = self.db_master_session
k1 = "ab1234567890"
k2 = "cd1234567890"
data = [
Wifi(key=k1, new_measures=2, total_measures=2),
WifiMeasure(lat=1.0, lon=1.0, key=k1),
WifiMeasure(lat=1.002, lon=1.004, key=k1),
Wifi(key=k2, lat=2.0, lon=-2.0,
max_lat=2.001, min_lat=1.999,
max_lon=-1.999, min_lon=-2.001,
new_measures=2, total_measures=4),
WifiMeasure(lat=2.002, lon=-2.004, key=k2),
WifiMeasure(lat=1.998, lon=-1.996, key=k2),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
wifis = dict(session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.001)
self.assertEqual(wifis[k1].max_lat, 1.002)
self.assertEqual(wifis[k1].min_lat, 1.0)
self.assertEqual(wifis[k1].lon, 1.002)
self.assertEqual(wifis[k1].max_lon, 1.004)
self.assertEqual(wifis[k1].min_lon, 1.0)
self.assertEqual(wifis[k2].lat, 2.0)
self.assertEqual(wifis[k2].max_lat, 2.002)
self.assertEqual(wifis[k2].min_lat, 1.998)
self.assertEqual(wifis[k2].lon, -2.0)
self.assertEqual(wifis[k2].max_lon, -1.996)
self.assertEqual(wifis[k2].min_lon, -2.004)
# independent calculation: the k1 bounding box is
# (1.000, 1.000) to (1.002, 1.004), with centroid
# at (1.001, 1.002); worst distance from centroid
# to any corner is 249m
self.assertEqual(wifis[k1].range, 249)
# independent calculation: the k2 bounding box is
# (1.998, -2.004) to (2.002, -1.996), with centroid
# at (2.000, 2.000); worst distance from centroid
# to any corner is 497m
self.assertEqual(wifis[k2].range, 497)
def test_location_update_wifi(self):
now = util.utcnow()
before = now - timedelta(days=1)
session = self.db_master_session
k1 = "ab1234567890"
k2 = "cd1234567890"
data = [
Wifi(key=k1, new_measures=3, total_measures=3),
WifiMeasure(lat=1.0, lon=1.0, key=k1),
WifiMeasure(lat=1.002, lon=1.003, key=k1),
WifiMeasure(lat=1.004, lon=1.006, key=k1),
Wifi(key=k2, lat=2.0, lon=2.0,
new_measures=2, total_measures=4),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
WifiMeasure(lat=-1.0, lon=-1.0, key=k2, created=before),
WifiMeasure(lat=-1.0, lon=-1.0, key=k2, created=before),
WifiMeasure(lat=2.002, lon=2.004, key=k2, created=now),
WifiMeasure(lat=2.002, lon=2.004, key=k2, created=now),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_wifi'],
gauge=['task.data.location_update_wifi.new_measures_1_100'],
)
wifis = dict(session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.002)
self.assertEqual(wifis[k1].lon, 1.003)
self.assertEqual(wifis[k1].new_measures, 0)
self.assertEqual(wifis[k2].lat, 2.001)
self.assertEqual(wifis[k2].lon, 2.002)
self.assertEqual(wifis[k2].new_measures, 0)
def test_location_update_wifi(self):
now = util.utcnow()
before = now - timedelta(days=1)
session = self.db_master_session
k1 = "ab1234567890"
k2 = "cd1234567890"
data = [
Wifi(key=k1, new_measures=3, total_measures=3),
WifiMeasure(lat=1.0, lon=1.0, key=k1),
WifiMeasure(lat=1.002, lon=1.003, key=k1),
WifiMeasure(lat=1.004, lon=1.006, key=k1),
Wifi(key=k2, lat=2.0, lon=2.0, new_measures=2, total_measures=4),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
WifiMeasure(lat=-1.0, lon=-1.0, key=k2, created=before),
WifiMeasure(lat=-1.0, lon=-1.0, key=k2, created=before),
WifiMeasure(lat=2.002, lon=2.004, key=k2, created=now),
WifiMeasure(lat=2.002, lon=2.004, key=k2, created=now),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_wifi'],
gauge=['task.data.location_update_wifi.new_measures_1_100'],
)
wifis = dict(session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.002)
self.assertEqual(wifis[k1].lon, 1.003)
self.assertEqual(wifis[k1].new_measures, 0)
self.assertEqual(wifis[k2].lat, 2.001)
self.assertEqual(wifis[k2].lon, 2.002)
self.assertEqual(wifis[k2].new_measures, 0)
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a wifi that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(id=month, key="ab1234567890",
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_wifi.delay(
[measure], utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_wifi.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(WifiBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(WifiMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# One wifi record should exist.
self.assertEqual(session.query(Wifi).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist.
self.assertEqual(update_result.get(), (1, 1))
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(WifiBlacklist).count(), 1)
self.assertEqual(session.query(Wifi).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0, insert_measures_wifi.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_wifis(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.db_master_session
k1 = "ab1234567890"
k2 = "bc1234567890"
k3 = "cd1234567890"
k4 = "de1234567890"
k5 = "ef1234567890"
k6 = "fa1234567890"
keys = set([k1, k2, k3, k4, k5, k6])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a wifi with an entry but no prior position
Wifi(key=k1, new_measures=3, total_measures=0),
WifiMeasure(lat=1.001, lon=1.001, key=k1),
WifiMeasure(lat=1.002, lon=1.005, key=k1),
WifiMeasure(lat=1.003, lon=1.009, key=k1),
# a wifi with a prior known position
Wifi(lat=2.0, lon=2.0, key=k2,
new_measures=2, total_measures=1),
WifiMeasure(lat=2.01, lon=2, key=k2),
WifiMeasure(lat=2.07, lon=2, key=k2),
# a wifi with a very different prior position
Wifi(lat=1.0, lon=1.0, key=k3,
new_measures=2, total_measures=1),
WifiMeasure(lat=3.0, lon=3.0, key=k3),
WifiMeasure(lat=-3.0, lon=3.0, key=k3),
# another wifi with a prior known position (and negative lat)
Wifi(lat=-4.0, lon=4.0, key=k4,
new_measures=2, total_measures=1),
WifiMeasure(lat=-4.1, lon=4, key=k4),
WifiMeasure(lat=-4.16, lon=4, key=k4),
# an already blacklisted wifi
WifiBlacklist(key=k5),
WifiMeasure(lat=5.0, lon=5.0, key=k5),
WifiMeasure(lat=5.1, lon=5.0, key=k5),
# a wifi with an old different record we ignore, position
# estimate has been updated since
Wifi(lat=6.0, lon=6.0, key=k6,
new_measures=2, total_measures=1),
WifiMeasure(lat=6.9, lon=6.9, key=k6, time=long_ago),
WifiMeasure(lat=6.0, lon=6.0, key=k6),
WifiMeasure(lat=6.001, lon=6.0, key=k6),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(WifiBlacklist).all()
self.assertEqual(set([b.key for b in black]), set([k2, k3, k4, k5]))
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 14)
self.assertEqual(set([m.key for m in measures]), keys)
# test duplicate call
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), 0)
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_wifi', 2),
# One of those would've scheduled a remove_wifi task
('task.data.remove_wifi', 1)
],
gauge=[
('task.data.location_update_wifi.new_measures_1_100', 2),
])
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a wifi that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(key="ab1234567890",
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_wifi.delay([measure],
utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_wifi.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(WifiBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(WifiMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# One wifi record should exist.
self.assertEqual(session.query(Wifi).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist.
self.assertEqual(update_result.get(), (1, 1))
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(WifiBlacklist).count(), 1)
self.assertEqual(session.query(Wifi).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0,
insert_measures_wifi.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_wifis(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.db_master_session
k1 = "ab1234567890"
k2 = "bc1234567890"
k3 = "cd1234567890"
k4 = "de1234567890"
k5 = "ef1234567890"
k6 = "fa1234567890"
keys = set([k1, k2, k3, k4, k5, k6])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a wifi with an entry but no prior position
Wifi(key=k1, new_measures=3, total_measures=0),
WifiMeasure(lat=1.001, lon=1.001, key=k1),
WifiMeasure(lat=1.002, lon=1.005, key=k1),
WifiMeasure(lat=1.003, lon=1.009, key=k1),
# a wifi with a prior known position
Wifi(lat=2.0, lon=2.0, key=k2, new_measures=2, total_measures=1),
WifiMeasure(lat=2.01, lon=2, key=k2),
WifiMeasure(lat=2.07, lon=2, key=k2),
# a wifi with a very different prior position
Wifi(lat=1.0, lon=1.0, key=k3, new_measures=2, total_measures=1),
WifiMeasure(lat=3.0, lon=3.0, key=k3),
WifiMeasure(lat=-3.0, lon=3.0, key=k3),
# another wifi with a prior known position (and negative lat)
Wifi(lat=-4.0, lon=4.0, key=k4, new_measures=2, total_measures=1),
WifiMeasure(lat=-4.1, lon=4, key=k4),
WifiMeasure(lat=-4.16, lon=4, key=k4),
# an already blacklisted wifi
WifiBlacklist(key=k5),
WifiMeasure(lat=5.0, lon=5.0, key=k5),
WifiMeasure(lat=5.1, lon=5.0, key=k5),
# a wifi with an old different record we ignore, position
# estimate has been updated since
Wifi(lat=6.0, lon=6.0, key=k6, new_measures=2, total_measures=1),
WifiMeasure(lat=6.9, lon=6.9, key=k6, time=long_ago),
WifiMeasure(lat=6.0, lon=6.0, key=k6),
WifiMeasure(lat=6.001, lon=6.0, key=k6),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(WifiBlacklist).all()
self.assertEqual(set([b.key for b in black]), set([k2, k3, k4, k5]))
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 14)
self.assertEqual(set([m.key for m in measures]), keys)
# test duplicate call
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), 0)
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_wifi', 2),
# One of those would've scheduled a remove_wifi task
('task.data.remove_wifi', 1)
],
gauge=[
('task.data.location_update_wifi.new_measures_1_100', 2),
])
