def test_schedule_cell_measures(self):
session = self.db_master_session
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
measures = []
for i in range(20):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival.delay(batch=15).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + 15))
blocks = schedule_cellmeasure_archival.delay(batch=6).get()
self.assertEquals(len(blocks), 0)
blocks = schedule_cellmeasure_archival.delay(batch=5).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id + 15, start_id + 20))
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
def test_schedule_cell_measures(self):
session = self.db_master_session
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
measures = []
for i in range(20):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival.delay(batch=15).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + 15))
blocks = schedule_cellmeasure_archival.delay(batch=6).get()
self.assertEquals(len(blocks), 0)
blocks = schedule_cellmeasure_archival.delay(batch=5).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id + 15, start_id + 20))
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
def test_schedule_cell_observations(self):
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
obs = CellObservationFactory.create_batch(20, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_cellmeasure_archival.delay(batch=15).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + 15))
blocks = schedule_cellmeasure_archival.delay(batch=6).get()
self.assertEquals(len(blocks), 0)
blocks = schedule_cellmeasure_archival.delay(batch=5).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id + 15, start_id + 20))
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
def test_schedule_cell_observations(self):
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
obs = CellObservationFactory.create_batch(20, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_cellmeasure_archival.delay(batch=15).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + 15))
blocks = schedule_cellmeasure_archival.delay(batch=6).get()
self.assertEquals(len(blocks), 0)
blocks = schedule_cellmeasure_archival.delay(batch=5).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id + 15, start_id + 20))
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
def test_backup_cell_to_s3(self):
session = self.db_master_session
batch_size = 10
measures = []
for i in range(batch_size):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend, 'backup_archive',
lambda x, y, z: True):
write_cellmeasure_s3_backups.delay(cleanup_zip=False).get()
msgs = self.heka_client.stream.msgs
info_msgs = [m for m in msgs if m.type == 'oldstyle']
self.assertEquals(1, len(info_msgs))
info = info_msgs[0]
fname = info.payload.split(":")[-1]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(
['alembic_revision.txt', 'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = session.query(MeasureBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/cell_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_backup_cell_to_s3(self):
session = self.db_master_session
batch_size = 10
measures = []
for i in range(batch_size):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
write_cellmeasure_s3_backups.delay(cleanup_zip=False).get()
msgs = self.heka_client.stream.msgs
info_msgs = [m for m in msgs if m.type == 'oldstyle']
self.assertEquals(1, len(info_msgs))
info = info_msgs[0]
fname = info.payload.split(":")[-1]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = session.query(MeasureBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/cell_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_backup_cell_to_s3(self):
batch_size = 10
obs = CellObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
write_cellmeasure_s3_backups.delay(cleanup_zip=False).get()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEqual(block.archive_sha, '20bytes_mean_success')
self.assertEqual(block.s3_key, 'skipped')
self.assertTrue(block.archive_date is None)
def test_backup_cell_to_s3(self):
batch_size = 10
obs = CellObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend, 'backup_archive',
lambda x, y, z: True):
write_cellmeasure_s3_backups.delay(cleanup_zip=False).get()
raven_msgs = self.raven_client.msgs
fname = [
m['message'].split(':')[1] for m in raven_msgs
if m['message'].startswith('s3.backup:')
][0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(
['alembic_revision.txt', 'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/cell_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_backup_cell_to_s3(self):
batch_size = 10
obs = CellObservationFactory.create_batch(batch_size, created=self.old)
self.session.flush()
start_id = obs[0].id
blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend,
'backup_archive', lambda x, y, z: True):
write_cellmeasure_s3_backups.delay(cleanup_zip=False).get()
raven_msgs = self.raven_client.msgs
fname = [m['message'].split(':')[1] for m in raven_msgs
if m['message'].startswith('s3.backup:')][0]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(['alembic_revision.txt',
'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = self.session.query(ObservationBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/cell_' in block.s3_key)
self.assertTrue(block.archive_date is None)