def test_schedule_cell_measures(self):
session = self.db_master_session
blocks = schedule_cellmeasure_archival(batch=1)
self.assertEquals(len(blocks), 0)
measures = []
for i in range(20):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival(batch=15)
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + 15))
blocks = schedule_cellmeasure_archival(batch=6)
self.assertEquals(len(blocks), 0)
blocks = schedule_cellmeasure_archival(batch=5)
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id + 15, start_id + 20))
blocks = schedule_cellmeasure_archival(batch=1)
self.assertEquals(len(blocks), 0)
def test_backup_cell_to_s3(self):
session = self.db_master_session
batch_size = 10
measures = []
for i in range(batch_size):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival(batch=batch_size)
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
write_cellmeasure_s3_backups(cleanup_zip=False)
msgs = self.heka_client.stream.msgs
info_msgs = [m for m in msgs if m.type == 'oldstyle']
self.assertEquals(1, len(info_msgs))
info = info_msgs[0]
fname = info.payload.split(":")[-1]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = session.query(MeasureBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/cell_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_backup_cell_to_s3(self):
session = self.db_master_session
batch_size = 10
measures = []
for i in range(batch_size):
measures.append(CellMeasure())
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival(batch=batch_size)
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
zips = write_cellmeasure_s3_backups(cleanup_zip=False)
self.assertTrue(len(zips), 1)
fname = zips[0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = session.query(MeasureBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue(block.archive_date is None)