def test_monitor_measures(self):
result = monitor_measures.delay().get()
self.check_stats(gauge=[('table.cell_measure', 1),
('table.wifi_measure', 1)], )
self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1})
# add some observations
CellObservationFactory.create_batch(3)
WifiObservationFactory.create_batch(5)
self.session.flush()
result = monitor_measures.delay().get()
self.check_stats(gauge=[('table.cell_measure', 2),
('table.wifi_measure', 2)], )
self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_monitor_measures(self):
result = monitor_measures.delay().get()
self.check_stats(
gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)],
)
self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1})
# add some observations
CellObservationFactory.create_batch(3)
WifiObservationFactory.create_batch(5)
self.session.flush()
result = monitor_measures.delay().get()
self.check_stats(
gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)],
)
self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_delete_wifi_observations(self):
obs = WifiObservationFactory.create_batch(50, created=self.old)
self.session.flush()
start_id = obs[0].id + 20
block = ObservationBlockFactory(
measure_type=ObservationType.wifi,
start_id=start_id, end_id=start_id + 20, archive_date=None)
self.session.commit()
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(self.session.query(WifiObservation).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_wifi_histogram(self):
session = self.session
today = util.utcnow()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(3))
WifiObservationFactory.create_batch(2, created=today)
WifiObservationFactory(created=yesterday)
WifiObservationFactory.create_batch(3, created=two_days)
WifiObservationFactory(created=long_ago)
session.flush()
wifi_histogram.delay(ago=3).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, StatKey.wifi)
self.assertEqual(stats[0].time, long_ago.date())
self.assertEqual(stats[0].value, 1)
# fill in newer dates
wifi_histogram.delay(ago=2).get()
wifi_histogram.delay(ago=1).get()
wifi_histogram.delay(ago=0).get()
# test duplicate execution
wifi_histogram.delay(ago=1).get()
stats = session.query(Stat.time, Stat.value).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(dict(stats), {
long_ago.date(): 1,
two_days.date(): 4,
yesterday.date(): 5,
today.date(): 7})
def test_delete_wifi_observations(self):
obs = WifiObservationFactory.create_batch(50, created=self.old)
self.session.flush()
start_id = obs[0].id + 20
block = ObservationBlockFactory(measure_type=ObservationType.wifi,
start_id=start_id,
end_id=start_id + 20,
archive_date=None)
self.session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(self.session.query(WifiObservation).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_backup_wifi_to_s3(self):
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEqual(block.archive_sha, '20bytes_mean_success')
self.assertEqual(block.s3_key, 'skipped')
self.assertTrue(block.archive_date is None)
def test_backup_wifi_to_s3(self):
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend, 'backup_archive',
lambda x, y, z: True):
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
raven_msgs = self.raven_client.msgs
fname = [
m['message'].split(':')[1] for m in raven_msgs
if m['message'].startswith('s3.backup:')
][0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(
['alembic_revision.txt', 'wifi_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/wifi_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_schedule_wifi_observations(self):
blocks = schedule_wifimeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size * 2,
created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 2)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
block = blocks[1]
self.assertEquals(block,
(start_id + batch_size, start_id + 2 * batch_size))
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 0)
def test_schedule_wifi_observations(self):
blocks = schedule_wifimeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
batch_size = 10
obs = WifiObservationFactory.create_batch(
batch_size * 2, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 2)
block = blocks[0]
self.assertEquals(block,
(start_id, start_id + batch_size))
block = blocks[1]
self.assertEquals(block,
(start_id + batch_size, start_id + 2 * batch_size))
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 0)
def test_backup_wifi_to_s3(self):
batch_size = 10
obs = WifiObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
write_wifimeasure_s3_backups.delay(cleanup_zip=False).get()
raven_msgs = self.raven_client.msgs
fname = [m['message'].split(':')[1] for m in raven_msgs
if m['message'].startswith('s3.backup:')][0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'wifi_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/wifi_' in block.s3_key)
self.assertTrue(block.archive_date is None)
