def test_stats(self):
self.session.add(ApiKey(valid_key='e5444-794', log=True))
self.session.flush()
self.add_reports(3, email='secretemail@localhost',
ip=self.geoip_data['London']['ip'])
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
schedule_export_reports.delay().get()
update_cell.delay().get()
update_wifi.delay().get()
self.check_stats(counter=[
('data.export.batch', 1, 1, ['key:internal']),
('data.report.upload', 2, 3),
('data.report.upload', 1, 3, ['key:test']),
('data.report.upload', 1, 6, ['key:e5444-794']),
('data.observation.insert', 1, 12, ['type:cell']),
('data.observation.insert', 1, 24, ['type:wifi']),
('data.observation.upload', 1, 3, ['type:cell', 'key:test']),
('data.observation.upload', 1, 6, ['type:wifi', 'key:test']),
('data.observation.upload', 0, ['type:cell', 'key:no_key']),
('data.observation.upload', 1, 6, ['type:cell', 'key:e5444-794']),
('data.observation.upload', 1, 12, ['type:wifi', 'key:e5444-794']),
])
def test_blacklist(self):
utcnow = util.utcnow()
bad_wifi = WifiFactory.build()
good_wifi = WifiFactory.build()
black = WifiBlacklist(time=utcnow, count=1, key=bad_wifi.key)
self.session.add(black)
self.session.flush()
obs = dict(lat=good_wifi.lat, lon=good_wifi.lon)
entries = [
{'key': good_wifi.key},
{'key': good_wifi.key},
{'key': bad_wifi.key},
]
for entry in entries:
entry.update(obs)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
self.assertEqual(self.data_queue.size(), 2)
update_wifi.delay().get()
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self._compare_sets([w.key for w in wifis], [good_wifi.key])
self.check_statcounter(StatKey.wifi, 2)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_country(self):
obs = []
obs_factory = WifiObservationFactory
wifi1 = WifiShardFactory(lat=55.807, lon=1.7465, country='GB')
obs.append(obs_factory(
key=wifi1.mac,
lat=wifi1.lat,
lon=wifi1.lon + 0.001,
))
wifi2 = WifiShardFactory(lat=50.022, lon=1.7465, country='GB')
obs.append(obs_factory(
key=wifi2.mac,
lat=wifi2.lat,
lon=wifi2.lon + 0.001,
))
self.data_queue.enqueue(obs)
self.session.commit()
update_wifi.delay().get()
wifi1 = self.session.query(wifi1.__class__).get(wifi1.mac)
self.assertEqual(wifi1.block_count, 0)
self.assertEqual(wifi1.country, None)
wifi2 = self.session.query(wifi2.__class__).get(wifi2.mac)
self.assertEqual(wifi1.block_count, 0)
self.assertEqual(wifi2.country, 'FR')
def test_temp_blocked_admitted_again(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1)
obs = WifiObservationFactory()
WifiShardFactory(
mac=obs.mac,
lat=None,
lon=None,
samples=0,
created=last_week,
modified=last_week,
block_first=last_week.date(),
block_last=last_week.date(),
block_count=1)
self.session.flush()
# add a new entry for the previously blocked wifi
self.data_queue.enqueue([obs])
self.assertEqual(self.data_queue.size(), 1)
update_wifi.delay().get()
# the wifi was inserted again
shard = WifiShard.shard_model(obs.mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertEqual(wifi.created.date(), last_week.date())
self.assertAlmostEqual(wifi.lat, obs.lat)
self.assertAlmostEqual(wifi.lon, obs.lon)
self.assertEqual(wifi.country, 'GB')
self.assertEqual(wifi.samples, 1)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_stats(self):
self.session.add(ApiKey(valid_key='e5444-794', log_submit=True))
self.session.flush()
self.add_reports(3, email='secretemail@localhost',
ip=self.geoip_data['London']['ip'])
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
schedule_export_reports.delay().get()
update_cell.delay().get()
for i in range(16):
update_wifi.delay(shard_id='%x' % i).get()
self.check_stats(counter=[
('data.export.batch', 1, 1, ['key:internal']),
('data.report.upload', 2, 3),
('data.report.upload', 1, 3, ['key:test']),
('data.report.upload', 1, 6, ['key:e5444-794']),
('data.observation.insert', 1, 12, ['type:cell']),
('data.observation.upload', 1, 3, ['type:cell', 'key:test']),
('data.observation.upload', 1, 6, ['type:wifi', 'key:test']),
('data.observation.upload', 0, ['type:cell', 'key:no_key']),
('data.observation.upload', 1, 6, ['type:cell', 'key:e5444-794']),
('data.observation.upload', 1, 12, ['type:wifi', 'key:e5444-794']),
])
# we get a variable number of statsd messages and are only
# interested in the sum-total
insert_msgs = [msg for msg in self.stats_client.msgs
if (msg.startswith('data.observation.insert') and
'type:wifi' in msg)]
self.assertEqual(
sum([int(msg.split(':')[1].split('|')[0]) for msg in insert_msgs]),
24)
def test_new(self):
utcnow = util.utcnow()
obs = WifiObservationFactory.build()
self.data_queue.enqueue([obs])
self.assertEqual(self.data_queue.size(), 1)
update_wifi.delay().get()
shard = WifiShard.shard_model(obs.mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertAlmostEqual(wifi.lat, obs.lat)
self.assertAlmostEqual(wifi.max_lat, obs.lat)
self.assertAlmostEqual(wifi.min_lat, obs.lat)
self.assertAlmostEqual(wifi.lon, obs.lon)
self.assertAlmostEqual(wifi.max_lon, obs.lon)
self.assertAlmostEqual(wifi.min_lon, obs.lon)
self.assertEqual(wifi.country, 'GB')
self.assertEqual(wifi.radius, 0)
self.assertEqual(wifi.samples, 1)
self.assertEqual(wifi.created.date(), utcnow.date())
self.assertEqual(wifi.modified.date(), utcnow.date())
self.assertEqual(wifi.block_first, None)
self.assertEqual(wifi.block_last, None)
self.assertEqual(wifi.block_count, None)
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_blacklist_moving_wifis(self):
now = util.utcnow()
k1 = 'ab1234567890'
k2 = 'bc1234567890'
k3 = 'cd1234567890'
k4 = 'de1234567890'
k5 = 'ef1234567890'
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a wifi with an entry but no prior position
Wifi(key=k1, total_measures=0),
WifiObservation(lat=1.001, lon=1.001, key=k1),
WifiObservation(lat=1.002, lon=1.005, key=k1),
WifiObservation(lat=1.003, lon=1.009, key=k1),
# a wifi with a prior known position
Wifi(lat=2.0, lon=2.0, key=k2, total_measures=1),
WifiObservation(lat=2.01, lon=2, key=k2),
WifiObservation(lat=2.07, lon=2, key=k2),
# a wifi with a very different prior position
Wifi(lat=1.0, lon=1.0, key=k3, total_measures=1),
WifiObservation(lat=3.0, lon=3.0, key=k3),
WifiObservation(lat=-3.0, lon=3.0, key=k3),
# another wifi with a prior known position (and negative lat)
Wifi(lat=-4.0, lon=4.0, key=k4, total_measures=1),
WifiObservation(lat=-4.1, lon=4, key=k4),
WifiObservation(lat=-4.16, lon=4, key=k4),
# an already blacklisted wifi
WifiBlacklist(key=k5, time=now, count=1),
WifiObservation(lat=5.0, lon=5.0, key=k5),
WifiObservation(lat=5.1, lon=5.0, key=k5),
]
observations = []
for obj in data:
if isinstance(obj, WifiObservation):
observations.append(obj)
else:
self.session.add(obj)
self.data_queue.enqueue(observations)
self.session.commit()
result = update_wifi.delay()
self.assertEqual(result.get(), (4, 3))
black = self.session.query(WifiBlacklist).all()
self.assertEqual(set([b.key for b in black]), set([k2, k3, k4, k5]))
# test duplicate call
result = update_wifi.delay()
self.assertEqual(result.get(), (0, 0))
self.check_stats(
timer=[
# We made duplicate calls
('task.data.update_wifi', 2),
# One of those would've scheduled a remove_wifi task
('task.data.remove_wifi', 1)
])
def _queue_and_update(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[WifiShard.shard_id(ob.mac)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_wifi_' + shard_id]
queue.enqueue(values)
update_wifi.delay(shard_id=shard_id).get()
def test_stats(self):
self.add_reports(3, api_key='test')
schedule_export_reports.delay().get()
update_cell.delay().get()
update_wifi.delay().get()
self.check_stats(
counter=[('items.api_log.test.uploaded.reports', 1, 3),
('items.uploaded.reports', 1, 3)])
def _queue_and_update(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[WifiShard.shard_id(ob.mac)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_wifi_' + shard_id]
queue.enqueue(values)
update_wifi.delay(shard_id=shard_id).get()
def test_wifi_invalid(self):
self.add_reports(cell_factor=0, wifi_factor=1, wifi_key='abcd')
schedule_export_reports.delay().get()
update_wifi.delay().get()
self.check_stats(counter=[
('data.report.upload', 1, 1, ['key:test']),
('data.report.drop', 1, 1, ['reason:malformed', 'key:test']),
('data.observation.drop', 1, 1,
['type:wifi', 'reason:malformed', 'key:test']),
])
def _update_all(self):
update_incoming.delay().get()
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_upload_wifi(self):
reports = self.add_reports(cell_factor=0, wifi_factor=1)
schedule_export_reports.delay().get()
update_wifi.delay().get()
position = reports[0]['position']
wifi_data = reports[0]['wifiAccessPoints'][0]
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertEqual(wifi.lat, position['latitude'])
self.assertEqual(wifi.lon, position['longitude'])
self.assertEqual(wifi.key, wifi_data['macAddress'])
self.assertEqual(wifi.total_measures, 1)
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name="internal", batch=0, schema="internal")
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name='internal', batch=0, schema='internal')
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_max_min_range_update(self):
wifi = WifiFactory(range=100, total_measures=4)
wifi_lat = wifi.lat
wifi_lon = wifi.lon
wifi.max_lat = wifi.lat + 0.001
wifi.min_lat = wifi.lat - 0.001
wifi.max_lon = wifi.lon + 0.001
wifi.min_lon = wifi.lon - 0.001
observations = [
WifiObservation(lat=wifi.lat + 0.002, lon=wifi.lon - 0.004,
key=wifi.key),
WifiObservation(lat=wifi.lat - 0.002, lon=wifi.lon + 0.01,
key=wifi.key),
]
self.data_queue.enqueue(observations)
self.session.commit()
self.assertEqual(update_wifi.delay().get(), (1, 0))
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertAlmostEqual(wifi.lat, wifi_lat)
self.assertAlmostEqual(wifi.max_lat, wifi_lat + 0.002)
self.assertAlmostEqual(wifi.min_lat, wifi_lat - 0.002)
self.assertAlmostEqual(wifi.lon, wifi_lon + 0.001)
self.assertAlmostEqual(wifi.max_lon, wifi_lon + 0.01)
self.assertAlmostEqual(wifi.min_lon, wifi_lon - 0.004)
self.assertEqual(wifi.range, 662)
self.assertEqual(wifi.total_measures, 6)
def test_wifi(self):
reports = self.add_reports(cell_factor=0, wifi_factor=1)
schedule_export_reports.delay().get()
update_wifi.delay().get()
position = reports[0]['position']
wifi_data = reports[0]['wifiAccessPoints'][0]
mac = wifi_data['macAddress']
shard = WifiShard.shard_model(mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertEqual(wifi.lat, position['latitude'])
self.assertEqual(wifi.lon, position['longitude'])
self.assertEqual(wifi.mac, wifi_data['macAddress'])
self.assertEqual(wifi.samples, 1)
def test_upload_duplicated_wifi(self):
self.add_reports(cell_factor=0, wifi_factor=1)
# duplicate the wifi entry inside the report
queue = self.celery_app.export_queues['internal']
items = queue.dequeue(queue.queue_key())
report = items[0]['report']
wifi = report['wifiAccessPoints'][0].copy()
report['wifiAccessPoints'].append(wifi)
report['wifiAccessPoints'][1]['signalStrength'] += 2
queue.enqueue(items, queue.queue_key())
schedule_export_reports.delay().get()
update_wifi.delay().get()
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(wifis[0].total_measures, 1)
def test_insert_observations(self):
session = self.session
time = util.utcnow() - timedelta(days=1)
wifi = WifiFactory(total_measures=0)
wifi2 = WifiFactory.build(total_measures=0)
user = User(nickname=u'test')
session.add(user)
session.flush()
obs = dict(
created=time, time=time, lat=wifi.lat, lon=wifi.lon,
accuracy=None, altitude=None, altitude_accuracy=None,
heading=52.9, speed=158.5,
)
entries = [
{'key': wifi.key, 'channel': 11, 'signal': -80},
{'key': wifi.key, 'channel': 3, 'signal': -90},
{'key': wifi.key, 'channel': 3, 'signal': -80},
{'key': wifi2.key, 'channel': 3, 'signal': -90},
]
for entry in entries:
entry.update(obs)
result = insert_measures_wifi.delay(entries, userid=user.id)
self.assertEqual(result.get(), 4)
self.assertEqual(self.data_queue.size(), 4)
update_wifi.delay().get()
self.session.refresh(wifi)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self._compare_sets([w.key for w in wifis], [wifi.key, wifi2.key])
self._compare_sets([w.total_measures for w in wifis], [1, 3])
score_queue = self.celery_app.data_queues['update_score']
scores = score_queue.dequeue()
self.assertEqual(len(scores), 1)
score = scores[0]
self.assertEqual(score['hashkey'].userid, user.id)
self.assertEqual(score['hashkey'].key, ScoreKey.new_wifi)
self.assertEqual(score['value'], 1)
self.check_statcounter(StatKey.wifi, 4)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_max_min_range_update(self):
k1 = 'ab1234567890'
k2 = 'cd1234567890'
data = [
Wifi(key=k1, total_measures=2),
WifiObservation(lat=1.0, lon=1.0, key=k1),
WifiObservation(lat=1.002, lon=1.004, key=k1),
Wifi(key=k2, lat=2.0, lon=-2.0,
max_lat=2.001, min_lat=1.999,
max_lon=-1.999, min_lon=-2.001,
total_measures=4),
WifiObservation(lat=2.002, lon=-2.004, key=k2),
WifiObservation(lat=1.998, lon=-1.996, key=k2),
]
observations = []
for obj in data:
if isinstance(obj, WifiObservation):
observations.append(obj)
else:
self.session.add(obj)
self.data_queue.enqueue(observations)
self.session.commit()
result = update_wifi.delay()
self.assertEqual(result.get(), (2, 0))
wifis = dict(self.session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.001)
self.assertEqual(wifis[k1].max_lat, 1.002)
self.assertEqual(wifis[k1].min_lat, 1.0)
self.assertEqual(wifis[k1].lon, 1.002)
self.assertEqual(wifis[k1].max_lon, 1.004)
self.assertEqual(wifis[k1].min_lon, 1.0)
self.assertEqual(wifis[k2].lat, 2.0)
self.assertEqual(wifis[k2].max_lat, 2.002)
self.assertEqual(wifis[k2].min_lat, 1.998)
self.assertEqual(wifis[k2].lon, -2.0)
self.assertEqual(wifis[k2].max_lon, -1.996)
self.assertEqual(wifis[k2].min_lon, -2.004)
# independent calculation: the k1 bounding box is
# (1.000, 1.000) to (1.002, 1.004), with centroid
# at (1.001, 1.002); worst distance from centroid
# to any corner is 249m
self.assertEqual(wifis[k1].range, 249)
# independent calculation: the k2 bounding box is
# (1.998, -2.004) to (2.002, -1.996), with centroid
# at (2.000, 2.000); worst distance from centroid
# to any corner is 497m
self.assertEqual(wifis[k2].range, 497)
def test_wifi_duplicated(self):
self.add_reports(cell_factor=0, wifi_factor=1)
# duplicate the wifi entry inside the report
queue = self.celery_app.export_queues['internal']
items = queue.dequeue(queue.queue_key())
report = items[0]['report']
wifi = report['wifiAccessPoints'][0]
mac = wifi['macAddress']
report['wifiAccessPoints'].append(wifi.copy())
report['wifiAccessPoints'].append(wifi.copy())
report['wifiAccessPoints'][1]['signalStrength'] += 2
report['wifiAccessPoints'][2]['signalStrength'] -= 2
queue.enqueue(items, queue.queue_key())
schedule_export_reports.delay().get()
update_wifi.delay().get()
shard = WifiShard.shard_model(mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(wifis[0].samples, 1)
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
wifi = WifiFactory.build()
self.session.add(WifiBlacklist(time=last_week, count=1, key=wifi.key))
self.session.flush()
# add a new entry for the previously blacklisted wifi
obs = dict(lat=wifi.lat, lon=wifi.lon, key=wifi.key)
insert_measures_wifi.delay([obs]).get()
self.assertEqual(self.data_queue.size(), 1)
update_wifi.delay().get()
# the wifi was inserted again
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(wifis[0].created, last_week)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_upload(self):
self.session.add(ApiKey(valid_key='e5444-794', log=True))
self.session.flush()
self.add_reports(3, email='secretemail@localhost',
ip=self.geoip_data['London']['ip'])
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
schedule_export_reports.delay().get()
update_cell.delay().get()
update_wifi.delay().get()
self.assertEqual(self.session.query(Cell).count(), 12)
self.assertEqual(self.session.query(Wifi).count(), 24)
self.check_stats(counter=[
('items.export.internal.batches', 1, 1),
('items.api_log.test.uploaded.cell_observations', 1, 3),
('items.api_log.test.uploaded.wifi_observations', 1, 6),
('items.api_log.no_key.uploaded.cell_observations', 0),
('items.api_log.e5444-794.uploaded.cell_observations', 1, 6),
('items.api_log.e5444-794.uploaded.wifi_observations', 1, 12),
])
def test_temp_blocked(self):
utcnow = util.utcnow()
bad_wifi = WifiObservationFactory.build()
good_wifi = WifiObservationFactory.build()
WifiShardFactory(
mac=bad_wifi.mac,
lat=None,
lon=None,
created=utcnow,
block_first=utcnow.date() - timedelta(days=10),
block_last=utcnow.date(),
block_count=1)
self.session.flush()
observations = [good_wifi, bad_wifi, good_wifi]
self.data_queue.enqueue(observations)
self.assertEqual(self.data_queue.size(), 3)
update_wifi.delay().get()
shard = WifiShard.shard_model(good_wifi.mac)
wifis = (self.session.query(shard)
.filter(shard.mac == good_wifi.mac)).all()
self.assertEqual(len(wifis), 1)
self.assertTrue(wifis[0].lat is not None)
self.assertTrue(wifis[0].lon is not None)
self.assertEqual(wifis[0].samples, 2)
shard = WifiShard.shard_model(bad_wifi.mac)
wifis = (self.session.query(shard)
.filter(shard.mac == bad_wifi.mac)).all()
self.assertEqual(len(wifis), 1)
self.assertTrue(wifis[0].lat is None)
self.assertTrue(wifis[0].lon is None)
self.check_statcounter(StatKey.wifi, 2)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_update_wifi(self):
wifis = WifiFactory.create_batch(2)
obs = []
obs_factory = WifiObservationFactory
# first wifi
wifi = wifis[0]
wifi.total_measures = 3
lat1, lon1 = (wifi.lat, wifi.lon)
obs.extend([
obs_factory(lat=wifi.lat,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.003, key=wifi.key),
obs_factory(lat=wifi.lat + 0.004,
lon=wifi.lon + 0.006, key=wifi.key),
])
wifi.lat = None
wifi.lon = None
# second wifi
wifi = wifis[1]
wifi.total_measures = 2
wifi.lat += 1.0
wifi.lon += 1.0
lat2, lon2 = (wifi.lat, wifi.lon)
obs.extend([
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.004, key=wifi.key),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.004, key=wifi.key),
])
self.data_queue.enqueue(obs)
self.session.commit()
result = update_wifi.delay()
self.assertEqual(result.get(), (2, 0))
self.check_stats(
timer=['task.data.update_wifi'],
)
found = dict(self.session.query(Wifi.key, Wifi).all())
self.assertEqual(set(found.keys()), set([wifis[0].key, wifis[1].key]))
self.assertAlmostEqual(found[wifis[0].key].lat, lat1 + 0.002)
self.assertAlmostEqual(found[wifis[0].key].lon, lon1 + 0.003)
self.assertAlmostEqual(found[wifis[1].key].lat, lat2 + 0.001)
self.assertAlmostEqual(found[wifis[1].key].lon, lon2 + 0.002)
def test_update_wifi(self):
obs = []
obs_factory = WifiObservationFactory
# first wifi
wifi1 = WifiFactory(lat=None, lon=None, total_measures=3)
new_pos = WifiFactory.build()
lat1, lon1 = (new_pos.lat, new_pos.lon)
obs.extend([
obs_factory(lat=lat1,
lon=lon1, key=wifi1.key),
obs_factory(lat=lat1 + 0.002,
lon=lon1 + 0.003, key=wifi1.key),
obs_factory(lat=lat1 + 0.004,
lon=lon1 + 0.006, key=wifi1.key),
])
# second wifi
wifi2 = WifiFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, total_measures=2)
lat2, lon2 = (wifi2.lat, wifi2.lon)
obs.extend([
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=wifi2.key),
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=wifi2.key),
])
self.data_queue.enqueue(obs)
self.session.flush()
result = update_wifi.delay()
self.assertEqual(result.get(), (2, 0))
self.check_stats(
timer=['task.data.update_wifi'],
)
self.session.refresh(wifi1)
self.session.refresh(wifi2)
found = dict(self.session.query(Wifi.key, Wifi).all())
self.assertEqual(set(found.keys()), set([wifi1.key, wifi2.key]))
self.assertAlmostEqual(found[wifi1.key].lat, lat1 + 0.002)
self.assertAlmostEqual(found[wifi1.key].lon, lon1 + 0.003)
self.assertAlmostEqual(found[wifi2.key].lat, lat2 + 0.001)
self.assertAlmostEqual(found[wifi2.key].lon, lon2 + 0.002)
def test_update_wifi(self):
now = util.utcnow()
k1 = 'ab1234567890'
k2 = 'cd1234567890'
data = [
Wifi(key=k1, total_measures=3),
WifiObservation(lat=1.0, lon=1.0, key=k1, created=now),
WifiObservation(lat=1.002, lon=1.003, key=k1, created=now),
WifiObservation(lat=1.004, lon=1.006, key=k1, created=now),
Wifi(key=k2, lat=2.0, lon=2.0, total_measures=2),
WifiObservation(lat=2.002, lon=2.004, key=k2, created=now),
WifiObservation(lat=2.002, lon=2.004, key=k2, created=now),
]
observations = []
for obj in data:
if isinstance(obj, WifiObservation):
observations.append(obj)
else:
self.session.add(obj)
self.data_queue.enqueue(observations)
self.session.commit()
result = update_wifi.delay()
self.assertEqual(result.get(), (2, 0))
self.check_stats(
timer=['task.data.update_wifi'],
)
wifis = dict(self.session.query(Wifi.key, Wifi).all())
self.assertEqual(set(wifis.keys()), set([k1, k2]))
self.assertEqual(wifis[k1].lat, 1.002)
self.assertEqual(wifis[k1].lon, 1.003)
self.assertEqual(wifis[k2].lat, 2.001)
self.assertEqual(wifis[k2].lon, 2.002)
def test_blacklist_moving_wifis(self):
now = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
moving = set()
wifis = WifiFactory.create_batch(4)
wifis.append(WifiFactory.build())
# a wifi with an entry but no prior position
wifi = wifis[0]
wifi.total_measures = 0
obs.extend([
obs_factory(lat=wifi.lat + 0.001,
lon=wifi.lon + 0.001, key=wifi.key),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.005, key=wifi.key),
obs_factory(lat=wifi.lat + 0.003,
lon=wifi.lon + 0.009, key=wifi.key),
])
wifi.lat = None
wifi.lon = None
# a wifi with a prior known position
wifi = wifis[1]
wifi.total_measures = 1
wifi.lat += 1.0
wifi.lon += 1.0
obs.extend([
obs_factory(lat=wifi.lat + 0.01,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat + 0.07,
lon=wifi.lon, key=wifi.key),
])
moving.add(wifi.hashkey())
# a wifi with a very different prior position
wifi = wifis[2]
wifi.total_measures = 1
obs.extend([
obs_factory(lat=wifi.lat + 2.0,
lon=wifi.lon + 2.0, key=wifi.key),
obs_factory(lat=wifi.lat - 4.0,
lon=wifi.lon + 2.0, key=wifi.key),
])
moving.add(wifi.hashkey())
# another wifi with a prior known position (and negative lat)
wifi = wifis[3]
wifi.total_measures = 1
wifi.lat *= -1.0
obs.extend([
obs_factory(lat=wifi.lat - 0.1,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat - 0.16,
lon=wifi.lon, key=wifi.key),
])
moving.add(wifi.hashkey())
# an already blacklisted wifi
wifi = wifis[4]
WifiBlacklistFactory(key=wifi.key, time=now, count=1)
obs.extend([
obs_factory(lat=wifi.lat,
lon=wifi.lon, key=wifi.key),
obs_factory(lat=wifi.lat + 0.1,
lon=wifi.lon, key=wifi.key),
])
moving.add(wifi.hashkey())
self.data_queue.enqueue(obs)
self.session.commit()
result = update_wifi.delay()
self.assertEqual(result.get(), (4, 3))
black = self.session.query(WifiBlacklist).all()
self.assertEqual(set([b.hashkey() for b in black]), moving)
# test duplicate call
result = update_wifi.delay()
self.assertEqual(result.get(), (0, 0))
self.check_stats(
timer=[
# We made duplicate calls
('task.data.update_wifi', 2),
# One of those would've scheduled a remove_wifi task
('task.data.remove_wifi', 1)
])
def test_upload_invalid_wifi(self):
self.add_reports(cell_factor=0, wifi_factor=1, wifi_key='abcd')
schedule_export_reports.delay().get()
update_wifi.delay().get()
self.assertEqual(self.session.query(Wifi).count(), 0)
def test_blacklist_temporary_and_permanent(self):
# This test simulates a wifi that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
obs = dict(key='ab1234567890',
time=time,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-observations, override
# utcnow to set creation date
insert_result = insert_measures_wifi.delay(
[obs], utcnow=time)
# update_result is (num-stations, num-moving-stations)
update_result = update_wifi.delay()
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = self.session.query(WifiBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# observation was supposedly inserted.
bl = bl[0]
bl.time = time
self.session.add(bl)
self.session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# observation was admitted.
self.assertEqual(insert_result.get(), 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# One wifi record should exist.
self.assertEqual(self.session.query(Wifi).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist.
self.assertEqual(update_result.get(), (1, 1))
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(
self.session.query(WifiBlacklist).count(), 1)
self.assertEqual(self.session.query(Wifi).count(), 0)
# Try adding one more observation 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = time + timedelta(days=1)
obs['time'] = next_day
self.assertEqual(
0, insert_measures_wifi.delay([obs],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to permanent mode,
# so no observation accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), (0, 0))
def test_blocklist_moving_wifis(self):
now = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
moving = set()
wifis = WifiShardFactory.create_batch(8)
wifis.append(WifiShardFactory.build())
# a wifi without an entry and disagreeing observations
wifi = wifis[-1]
obs.extend([
obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with an entry but no prior position
wifi = wifis[0]
obs.extend([
obs_factory(lat=wifi.lat + 0.001,
lon=wifi.lon + 0.001, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.005, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.003,
lon=wifi.lon + 0.009, key=wifi.mac),
])
wifi.lat = None
wifi.lon = None
wifi.samples = 0
# a wifi with a prior known position
wifi = wifis[1]
wifi.samples = 1
wifi.lat += 1.0
wifi.lon += 1.0
obs.extend([
obs_factory(lat=wifi.lat + 0.01,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.07,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with a very different prior position
wifi = wifis[2]
wifi.samples = 1
obs.extend([
obs_factory(lat=wifi.lat + 2.0,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 2.002,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# another wifi with a prior known position (and negative lat)
wifi = wifis[3]
wifi.samples = 1
wifi.lat *= -1.0
obs.extend([
obs_factory(lat=wifi.lat - 0.1,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat - 0.16,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# an already blocked wifi
wifi = wifis[4]
wifi.block_last = now.date()
wifi.block_count = 1
obs.extend([
obs_factory(lat=wifi.lat,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.1,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a permanently blocked wifi
wifi = wifis[5]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a no longer blocked wifi
wifi = wifis[6]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
])
# a no longer blocked wifi with disagreeing observations
wifi = wifis[7]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, key=wifi.mac),
])
moving.add(wifi.mac)
self.data_queue.enqueue(obs)
self.session.commit()
update_wifi.delay().get()
shards = set()
for mac in moving:
shards.add(WifiShard.shard_model(mac))
blocks = []
for shard in shards:
for row in self.session.query(shard).all():
if row.blocked():
blocks.append(row)
self.assertEqual(set([b.mac for b in blocks]), moving)
self.check_stats(counter=[
('data.observation.drop', 1, 3,
['type:wifi', 'reason:blocklisted']),
('data.station.blocklist', 1, 5,
['type:wifi', 'action:add', 'reason:moving']),
])
def test_update(self):
utcnow = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
# first wifi
wifi1 = WifiShardFactory(lat=None, lon=None, samples=3)
new_pos = WifiShardFactory.build()
mac1, lat1, lon1 = (wifi1.mac, new_pos.lat, new_pos.lon)
obs.extend([
obs_factory(lat=lat1,
lon=lon1, key=mac1),
obs_factory(lat=lat1 + 0.002,
lon=lon1 + 0.003, key=mac1),
obs_factory(lat=lat1 + 0.004,
lon=lon1 + 0.006, key=mac1),
])
# second wifi
wifi2 = WifiShardFactory(
lat=lat1 + 1.0, lon=lon1 + 1.0,
max_lat=lat1 + 1.0, min_lat=lat1 + 0.999,
max_lon=lon1 + 1.0, min_lon=None,
radius=20, samples=2,
created=utcnow - timedelta(10),
modified=utcnow - timedelta(10))
mac2, lat2, lon2 = (wifi2.mac, wifi2.lat, wifi2.lon)
obs.extend([
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=mac2),
obs_factory(lat=lat2 + 0.002,
lon=lon2 + 0.004, key=mac2),
])
self.data_queue.enqueue(obs)
self.session.commit()
update_wifi.delay().get()
shard = WifiShard.shard_model(mac1)
found = self.session.query(shard).filter(shard.mac == mac1).one()
self.assertAlmostEqual(found.lat, lat1 + 0.002)
self.assertAlmostEqual(found.max_lat, lat1 + 0.004)
self.assertAlmostEqual(found.min_lat, lat1)
self.assertAlmostEqual(found.lon, lon1 + 0.003)
self.assertAlmostEqual(found.max_lon, lon1 + 0.006)
self.assertAlmostEqual(found.min_lon, lon1)
self.assertEqual(found.modified.date(), utcnow.date())
self.assertEqual(found.country, 'GB')
self.assertEqual(found.radius, 304)
self.assertEqual(found.samples, 6)
shard = WifiShard.shard_model(mac2)
found = self.session.query(shard).filter(shard.mac == mac2).one()
self.assertAlmostEqual(found.lat, lat2 + 0.001)
self.assertAlmostEqual(found.max_lat, lat2 + 0.002)
self.assertAlmostEqual(found.min_lat, lat2 - 0.001)
self.assertAlmostEqual(found.lon, lon2 + 0.002)
self.assertAlmostEqual(found.max_lon, lon2 + 0.004)
self.assertAlmostEqual(found.min_lon, lon2)
self.assertEqual(found.created.date(), utcnow.date() - timedelta(10))
self.assertEqual(found.modified.date(), utcnow.date())
self.assertEqual(found.country, 'GB')
self.assertEqual(found.radius, 260)
self.assertEqual(found.samples, 4)