def test_temp_blocked(self):
utcnow = util.utcnow()
bad_wifi = WifiObservationFactory.build()
good_wifi = WifiObservationFactory.build()
WifiShardFactory(
mac=bad_wifi.mac,
lat=None,
lon=None,
created=utcnow,
block_first=utcnow.date() - timedelta(days=10),
block_last=utcnow.date(),
block_count=1)
obs = [good_wifi, bad_wifi, good_wifi]
self.session.commit()
self._queue_and_update(obs)
shard = WifiShard.shard_model(good_wifi.mac)
wifis = (self.session.query(shard)
.filter(shard.mac == good_wifi.mac)).all()
self.assertEqual(len(wifis), 1)
self.assertTrue(wifis[0].lat is not None)
self.assertTrue(wifis[0].lon is not None)
self.assertEqual(wifis[0].samples, 2)
shard = WifiShard.shard_model(bad_wifi.mac)
wifis = (self.session.query(shard)
.filter(shard.mac == bad_wifi.mac)).all()
self.assertEqual(len(wifis), 1)
self.assertTrue(wifis[0].block_first < utcnow.date())
self.assertTrue(wifis[0].lat is None)
self.assertTrue(wifis[0].lon is None)
self.check_statcounter(StatKey.wifi, 2)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_temp_blocked(self):
utcnow = util.utcnow()
bad_wifi = WifiObservationFactory.build()
good_wifi = WifiObservationFactory.build()
WifiShardFactory(mac=bad_wifi.mac,
lat=None,
lon=None,
created=utcnow,
block_first=utcnow.date() - timedelta(days=10),
block_last=utcnow.date(),
block_count=1)
obs = [good_wifi, bad_wifi, good_wifi]
self.session.commit()
self._queue_and_update(obs)
shard = WifiShard.shard_model(good_wifi.mac)
wifis = (self.session.query(shard).filter(
shard.mac == good_wifi.mac)).all()
self.assertEqual(len(wifis), 1)
self.assertTrue(wifis[0].lat is not None)
self.assertTrue(wifis[0].lon is not None)
self.assertEqual(wifis[0].samples, 2)
shard = WifiShard.shard_model(bad_wifi.mac)
wifis = (self.session.query(shard).filter(
shard.mac == bad_wifi.mac)).all()
self.assertEqual(len(wifis), 1)
self.assertTrue(wifis[0].block_first < utcnow.date())
self.assertTrue(wifis[0].lat is None)
self.assertTrue(wifis[0].lon is None)
self.check_statcounter(StatKey.wifi, 2)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_monitor_measures(self):
result = monitor_measures.delay().get()
self.check_stats(gauge=[('table.cell_measure', 1),
('table.wifi_measure', 1)], )
self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1})
# add some observations
CellObservationFactory.create_batch(3)
WifiObservationFactory.create_batch(5)
self.session.flush()
result = monitor_measures.delay().get()
self.check_stats(gauge=[('table.cell_measure', 2),
('table.wifi_measure', 2)], )
self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_new(self):
utcnow = util.utcnow()
obs = WifiObservationFactory.build()
self.data_queue.enqueue([obs])
self.assertEqual(self.data_queue.size(), 1)
update_wifi.delay().get()
shard = WifiShard.shard_model(obs.mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertAlmostEqual(wifi.lat, obs.lat)
self.assertAlmostEqual(wifi.max_lat, obs.lat)
self.assertAlmostEqual(wifi.min_lat, obs.lat)
self.assertAlmostEqual(wifi.lon, obs.lon)
self.assertAlmostEqual(wifi.max_lon, obs.lon)
self.assertAlmostEqual(wifi.min_lon, obs.lon)
self.assertEqual(wifi.country, 'GB')
self.assertEqual(wifi.radius, 0)
self.assertEqual(wifi.samples, 1)
self.assertEqual(wifi.created.date(), utcnow.date())
self.assertEqual(wifi.modified.date(), utcnow.date())
self.assertEqual(wifi.block_first, None)
self.assertEqual(wifi.block_last, None)
self.assertEqual(wifi.block_count, None)
def test_temp_blocked_admitted_again(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1)
obs = WifiObservationFactory()
WifiShardFactory(mac=obs.mac,
lat=None,
lon=None,
samples=0,
created=last_week,
modified=last_week,
block_first=last_week.date(),
block_last=last_week.date(),
block_count=1)
self.session.commit()
# add a new entry for the previously blocked wifi
self._queue_and_update([obs])
# the wifi was inserted again
shard = WifiShard.shard_model(obs.mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertEqual(wifi.block_first, last_week.date())
self.assertEqual(wifi.block_last, last_week.date())
self.assertEqual(wifi.created.date(), last_week.date())
self.assertAlmostEqual(wifi.lat, obs.lat)
self.assertAlmostEqual(wifi.lon, obs.lon)
self.assertEqual(wifi.region, 'GB')
self.assertEqual(wifi.samples, 1)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_monitor_measures(self):
result = monitor_measures.delay().get()
self.check_stats(
gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)],
)
self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1})
# add some observations
CellObservationFactory.create_batch(3)
WifiObservationFactory.create_batch(5)
self.session.flush()
result = monitor_measures.delay().get()
self.check_stats(
gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)],
)
self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_json(self):
obs = WifiObservationFactory.build(accuracy=None)
result = WifiObservation.from_json(simplejson.loads(
simplejson.dumps(obs.to_json())))
self.assertTrue(type(result), WifiObservation)
self.assertTrue(result.accuracy is None)
self.assertEqual(result.mac, obs.mac)
self.assertEqual(result.lat, obs.lat)
self.assertEqual(result.lon, obs.lon)
def test_json(self):
obs = WifiObservationFactory.build(accuracy=None, source=ReportSource.query)
result = WifiObservation.from_json(json.loads(json.dumps(obs.to_json())))
assert type(result) is WifiObservation
assert result.accuracy is None
assert result.mac == obs.mac
assert result.lat == obs.lat
assert result.lon == obs.lon
assert result.source == ReportSource.query
assert type(result.source) is ReportSource
def test_json(self):
obs = WifiObservationFactory.build(
accuracy=None, source=ReportSource.query)
result = WifiObservation.from_json(simplejson.loads(
simplejson.dumps(obs.to_json())))
assert type(result) is WifiObservation
assert result.accuracy is None
assert result.mac == obs.mac
assert result.lat == obs.lat
assert result.lon == obs.lon
assert result.source == ReportSource.query
assert type(result.source) is ReportSource
def test_delete_wifi_observations(self):
obs = WifiObservationFactory.create_batch(50, created=self.old)
self.session.flush()
start_id = obs[0].id + 20
block = ObservationBlockFactory(
measure_type=ObservationType.wifi,
start_id=start_id, end_id=start_id + 20, archive_date=None)
self.session.commit()
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(self.session.query(WifiObservation).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_delete_wifi_observations(self):
obs = WifiObservationFactory.create_batch(50, created=self.old)
self.session.flush()
start_id = obs[0].id + 20
block = ObservationBlockFactory(measure_type=ObservationType.wifi,
start_id=start_id,
end_id=start_id + 20,
archive_date=None)
self.session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(self.session.query(WifiObservation).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_backup_wifi_to_s3(self):
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEqual(block.archive_sha, '20bytes_mean_success')
self.assertEqual(block.s3_key, 'skipped')
self.assertTrue(block.archive_date is None)
def test_backup_wifi_to_s3(self):
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend, 'backup_archive',
lambda x, y, z: True):
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
raven_msgs = self.raven_client.msgs
fname = [
m['message'].split(':')[1] for m in raven_msgs
if m['message'].startswith('s3.backup:')
][0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(
['alembic_revision.txt', 'wifi_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/wifi_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_schedule_wifi_observations(self):
blocks = schedule_wifimeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size * 2,
created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 2)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
block = blocks[1]
self.assertEquals(block,
(start_id + batch_size, start_id + 2 * batch_size))
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 0)
def test_backup_wifi_to_s3(self):
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
raven_msgs = self.raven_client.msgs
fname = [m['message'].split(':')[1] for m in raven_msgs
if m['message'].startswith('s3.backup:')][0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'wifi_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/wifi_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_schedule_wifi_observations(self):
blocks = schedule_wifimeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
batch_size = 10
obs = WifiObservationFactory.create_batch(
batch_size * 2, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 2)
block = blocks[0]
self.assertEquals(block,
(start_id, start_id + batch_size))
block = blocks[1]
self.assertEquals(block,
(start_id + batch_size, start_id + 2 * batch_size))
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 0)
def test_new(self):
utcnow = util.utcnow()
obs = WifiObservationFactory.build()
self._queue_and_update([obs])
shard = WifiShard.shard_model(obs.mac)
wifis = self.session.query(shard).all()
self.assertEqual(len(wifis), 1)
wifi = wifis[0]
self.assertAlmostEqual(wifi.lat, obs.lat)
self.assertAlmostEqual(wifi.max_lat, obs.lat)
self.assertAlmostEqual(wifi.min_lat, obs.lat)
self.assertAlmostEqual(wifi.lon, obs.lon)
self.assertAlmostEqual(wifi.max_lon, obs.lon)
self.assertAlmostEqual(wifi.min_lon, obs.lon)
self.assertEqual(wifi.radius, 0)
self.assertEqual(wifi.region, 'GB')
self.assertEqual(wifi.samples, 1)
self.assertEqual(wifi.created.date(), utcnow.date())
self.assertEqual(wifi.modified.date(), utcnow.date())
self.assertEqual(wifi.block_first, None)
self.assertEqual(wifi.block_last, None)
self.assertEqual(wifi.block_count, None)
def test_wifi_histogram(self):
session = self.session
today = util.utcnow()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(3))
WifiObservationFactory.create_batch(2, created=today)
WifiObservationFactory(created=yesterday)
WifiObservationFactory.create_batch(3, created=two_days)
WifiObservationFactory(created=long_ago)
session.flush()
wifi_histogram.delay(ago=3).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, StatKey.wifi)
self.assertEqual(stats[0].time, long_ago.date())
self.assertEqual(stats[0].value, 1)
# fill in newer dates
wifi_histogram.delay(ago=2).get()
wifi_histogram.delay(ago=1).get()
wifi_histogram.delay(ago=0).get()
# test duplicate execution
wifi_histogram.delay(ago=1).get()
stats = session.query(Stat.time, Stat.value).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(dict(stats), {
long_ago.date(): 1,
two_days.date(): 4,
yesterday.date(): 5,
today.date(): 7})
