def test_remove_wifi(self):
observations = []
wifi_keys = []
m1 = 1.0
m2 = 2.0
for key in ['a%s1234567890' % i for i in range(5)]:
wifi = Wifi(key=key)
wifi_keys.append(wifi.hashkey())
observations.append(wifi)
observations.append(WifiObservation(lat=m1, lon=m1, key=key))
observations.append(WifiObservation(lat=m2, lon=m2, key=key))
self.session.add_all(observations)
self.session.flush()
result = remove_wifi.delay(wifi_keys[:2])
self.assertEqual(result.get(), 2)
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 3)
result = remove_wifi.delay(wifi_keys)
self.assertEqual(result.get(), 3)
result = remove_wifi.delay(wifi_keys)
self.assertEqual(result.get(), 0)
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 0)
def test_remove_wifi(self):
session = self.session
observations = []
wifi_keys = [{'key': "a%s1234567890" % i} for i in range(5)]
m1 = 1.0
m2 = 2.0
for key in wifi_keys:
key = key['key']
observations.append(Wifi(key=key))
observations.append(WifiObservation(lat=m1, lon=m1, key=key))
observations.append(WifiObservation(lat=m2, lon=m2, key=key))
session.add_all(observations)
session.flush()
result = remove_wifi.delay(wifi_keys[:2])
self.assertEqual(result.get(), 2)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 3)
result = remove_wifi.delay(wifi_keys)
self.assertEqual(result.get(), 3)
result = remove_wifi.delay(wifi_keys)
self.assertEqual(result.get(), 0)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 0)
def test_max_min_range_update(self):
session = self.session
k1 = "ab1234567890"
k2 = "cd1234567890"
data = [
Wifi(key=k1, new_measures=2, total_measures=2),
WifiObservation(lat=1.0, lon=1.0, key=k1),
WifiObservation(lat=1.002, lon=1.004, key=k1),
Wifi(key=k2,
lat=2.0,
lon=-2.0,
max_lat=2.001,
min_lat=1.999,
max_lon=-1.999,
min_lon=-2.001,
new_measures=2,
total_measures=4),
WifiObservation(lat=2.002, lon=-2.004, key=k2),
WifiObservation(lat=1.998, lon=-1.996, key=k2),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
wifis = dict(session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.001)
self.assertEqual(wifis[k1].max_lat, 1.002)
self.assertEqual(wifis[k1].min_lat, 1.0)
self.assertEqual(wifis[k1].lon, 1.002)
self.assertEqual(wifis[k1].max_lon, 1.004)
self.assertEqual(wifis[k1].min_lon, 1.0)
self.assertEqual(wifis[k2].lat, 2.0)
self.assertEqual(wifis[k2].max_lat, 2.002)
self.assertEqual(wifis[k2].min_lat, 1.998)
self.assertEqual(wifis[k2].lon, -2.0)
self.assertEqual(wifis[k2].max_lon, -1.996)
self.assertEqual(wifis[k2].min_lon, -2.004)
# independent calculation: the k1 bounding box is
# (1.000, 1.000) to (1.002, 1.004), with centroid
# at (1.001, 1.002); worst distance from centroid
# to any corner is 249m
self.assertEqual(wifis[k1].range, 249)
# independent calculation: the k2 bounding box is
# (1.998, -2.004) to (2.002, -1.996), with centroid
# at (2.000, 2.000); worst distance from centroid
# to any corner is 497m
self.assertEqual(wifis[k2].range, 497)
def test_schedule_wifi_observations(self):
session = self.db_master_session
blocks = schedule_wifimeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
batch_size = 10
observations = []
for i in range(batch_size * 2):
observations.append(WifiObservation(created=self.really_old))
session.add_all(observations)
session.flush()
start_id = observations[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 2)
block = blocks[0]
self.assertEquals(block,
(start_id, start_id + batch_size))
block = blocks[1]
self.assertEquals(block,
(start_id + batch_size, start_id + 2 * batch_size))
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 0)
def test_unthrottle_wifi_observations(self):
session = self.db_master_session
block = ObservationBlock()
block.measure_type = ObservationType.wifi
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
k = dict(lat=1.0, lon=1.0)
for i in range(100, 150):
session.add(
WifiObservation(id=i, key=str(i), created=self.really_old))
session.add(Wifi(total_measures=11000, key=str(i), **k))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
wifi_unthrottle_measures.delay(10000, 1000).get()
wifis = session.query(Wifi).all()
self.assertEquals(len(wifis), 50)
for wifi in wifis:
if 120 <= int(wifi.key) and int(wifi.key) < 140:
self.assertEquals(wifi.total_measures, 0)
else:
self.assertEquals(wifi.total_measures, 1)
self.check_stats(counter=['items.wifi_unthrottled'])
def test_backup_wifi_to_s3(self):
session = self.db_master_session
batch_size = 10
observations = []
for i in range(batch_size):
observations.append(WifiObservation(created=self.really_old))
session.add_all(observations)
session.flush()
start_id = observations[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
msgs = self.heka_client.stream.msgs
info_msgs = [m for m in msgs if m.type == 'oldstyle']
self.assertEquals(1, len(info_msgs))
info = info_msgs[0]
fname = info.payload.split(":")[-1]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'wifi_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/wifi_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_monitor_measures(self):
session = self.db_master_session
result = monitor_measures.delay().get()
self.check_stats(
gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)],
)
self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1})
# add some observations
session.add_all([CellObservation() for i in range(3)])
session.add_all([WifiObservation() for i in range(5)])
session.flush()
result = monitor_measures.delay().get()
self.check_stats(
gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)],
)
self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_delete_wifi_observations(self):
session = self.db_master_session
block = ObservationBlock()
block.measure_type = ObservationType.wifi
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
for i in range(100, 150):
session.add(WifiObservation(id=i, created=self.really_old))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(session.query(WifiObservation).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_wifi_histogram(self):
session = self.db_master_session
today = util.utcnow().date()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(3))
observations = [
WifiObservation(lat=1.0, lon=2.0, created=today),
WifiObservation(lat=1.0, lon=2.0, created=today),
WifiObservation(lat=1.0, lon=2.0, created=yesterday),
WifiObservation(lat=1.0, lon=2.0, created=two_days),
WifiObservation(lat=1.0, lon=2.0, created=two_days),
WifiObservation(lat=1.0, lon=2.0, created=two_days),
WifiObservation(lat=1.0, lon=2.0, created=long_ago),
]
session.add_all(observations)
session.commit()
wifi_histogram.delay(ago=3).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, StatKey.wifi)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
# fill in newer dates
wifi_histogram.delay(ago=2).get()
wifi_histogram.delay(ago=1).get()
wifi_histogram.delay(ago=0).get()
# test duplicate execution
wifi_histogram.delay(ago=1).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
self.assertEqual(stats[1].time, two_days)
self.assertEqual(stats[1].value, 4)
self.assertEqual(stats[2].time, yesterday)
self.assertEqual(stats[2].value, 5)
self.assertEqual(stats[3].time, today)
self.assertEqual(stats[3].value, 7)
def test_location_update_wifi(self):
now = util.utcnow()
before = now - timedelta(days=1)
session = self.session
k1 = "ab1234567890"
k2 = "cd1234567890"
data = [
Wifi(key=k1, new_measures=3, total_measures=3),
WifiObservation(lat=1.0, lon=1.0, key=k1, created=now),
WifiObservation(lat=1.002, lon=1.003, key=k1, created=now),
WifiObservation(lat=1.004, lon=1.006, key=k1, created=now),
Wifi(key=k2, lat=2.0, lon=2.0, new_measures=2, total_measures=4),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old observations are skipped
WifiObservation(lat=-1.0, lon=-1.0, key=k2, created=before),
WifiObservation(lat=-1.0, lon=-1.0, key=k2, created=before),
WifiObservation(lat=2.002, lon=2.004, key=k2, created=now),
WifiObservation(lat=2.002, lon=2.004, key=k2, created=now),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_wifi'],
gauge=['task.data.location_update_wifi.new_measures_1_100'],
)
wifis = dict(session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.002)
self.assertEqual(wifis[k1].lon, 1.003)
self.assertEqual(wifis[k1].new_measures, 0)
self.assertEqual(wifis[k2].lat, 2.001)
self.assertEqual(wifis[k2].lon, 2.002)
self.assertEqual(wifis[k2].new_measures, 0)
def test_blacklist_moving_wifis(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.session
k1 = "ab1234567890"
k2 = "bc1234567890"
k3 = "cd1234567890"
k4 = "de1234567890"
k5 = "ef1234567890"
k6 = "fa1234567890"
keys = set([k1, k2, k3, k4, k5, k6])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a wifi with an entry but no prior position
Wifi(key=k1, new_measures=3, total_measures=0),
WifiObservation(lat=1.001, lon=1.001, key=k1),
WifiObservation(lat=1.002, lon=1.005, key=k1),
WifiObservation(lat=1.003, lon=1.009, key=k1),
# a wifi with a prior known position
Wifi(lat=2.0, lon=2.0, key=k2, new_measures=2, total_measures=1),
WifiObservation(lat=2.01, lon=2, key=k2),
WifiObservation(lat=2.07, lon=2, key=k2),
# a wifi with a very different prior position
Wifi(lat=1.0, lon=1.0, key=k3, new_measures=2, total_measures=1),
WifiObservation(lat=3.0, lon=3.0, key=k3),
WifiObservation(lat=-3.0, lon=3.0, key=k3),
# another wifi with a prior known position (and negative lat)
Wifi(lat=-4.0, lon=4.0, key=k4, new_measures=2, total_measures=1),
WifiObservation(lat=-4.1, lon=4, key=k4),
WifiObservation(lat=-4.16, lon=4, key=k4),
# an already blacklisted wifi
WifiBlacklist(key=k5, time=now, count=1),
WifiObservation(lat=5.0, lon=5.0, key=k5),
WifiObservation(lat=5.1, lon=5.0, key=k5),
# a wifi with an old different record we ignore, position
# estimate has been updated since
Wifi(lat=6.0, lon=6.0, key=k6, new_measures=2, total_measures=1),
WifiObservation(lat=6.9, lon=6.9, key=k6, time=long_ago),
WifiObservation(lat=6.0, lon=6.0, key=k6),
WifiObservation(lat=6.001, lon=6.0, key=k6),
]
session.add_all(data)
session.commit()
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(WifiBlacklist).all()
self.assertEqual(set([b.key for b in black]), set([k2, k3, k4, k5]))
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 14)
self.assertEqual(set([o.key for o in observations]), keys)
# test duplicate call
result = location_update_wifi.delay(min_new=1)
self.assertEqual(result.get(), (0, 0))
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_wifi', 2),
# One of those would've scheduled a remove_wifi task
('task.data.remove_wifi', 1)
],
gauge=[
('task.data.location_update_wifi.new_measures_1_100', 2),
])
