def testRestoreTimeout(self):
"""Verifies the persistor will reattempt failed object store writes after a timeout"""
backup_dir = tempfile.mkdtemp()
persistor = LogBatchPersistor(backup_dir=backup_dir)
batches = [
LogBatch('Log batch buffer 1A', ObjectStore.SERVER_LOG, 'test1',
'keyA'),
LogBatch('Log batch buffer 2B', ObjectStore.SERVER_LOG, 'test2',
'keyB'),
LogBatch('Log batch buffer 3C', ObjectStore.SERVER_LOG, 'test3',
'keyC')
]
persistor._RESTORE_INTERVAL_SECS = 0.100
# The "bad" object store which does nothing with puts.
oldStore = ObjectStore.GetInstance(ObjectStore.SERVER_LOG)
ObjectStore.SetInstance(
ObjectStore.SERVER_LOG,
_BadObjectStore(ObjectStore.SERVER_LOG_BUCKET,
temporary=True,
fail_fast=True))
for batch in batches:
persistor.PersistLogBatch(batch)
self.io_loop.add_callback(
partial(self._VerifyBackupBatches, backup_dir, batches))
# Reinstate the "good" object store.
ObjectStore.SetInstance(ObjectStore.SERVER_LOG, oldStore)
self._RunAsync(self.io_loop.add_timeout, time.time() + 0.200)
self._RunAsync(self._VerifyObjStoreBatches, batches)
def testBadObjStore(self):
"""Tests backup storage in case the object store is down. Also verifies close() method."""
backup_dir = tempfile.mkdtemp()
persistor = LogBatchPersistor(backup_dir=backup_dir)
batches = [
LogBatch('Log batch buffer 1A', ObjectStore.SERVER_LOG, 'test1',
'keyA'),
LogBatch('Log batch buffer 2B', ObjectStore.SERVER_LOG, 'test2',
'keyB'),
LogBatch('Log batch buffer 3C', ObjectStore.SERVER_LOG, 'test3',
'keyC'),
LogBatch('Log batch buffer 4D', ObjectStore.USER_LOG, 'test4',
'keyD'),
LogBatch('Log batch buffer 5E', ObjectStore.USER_LOG, 'test5',
'keyE')
]
oldStores = [
ObjectStore.GetInstance(ObjectStore.SERVER_LOG),
ObjectStore.GetInstance(ObjectStore.USER_LOG)
]
ObjectStore.SetInstance(
ObjectStore.SERVER_LOG,
_BadObjectStore(ObjectStore.SERVER_LOG_BUCKET,
temporary=True,
fail_fast=False))
ObjectStore.SetInstance(
ObjectStore.USER_LOG,
_BadObjectStore(ObjectStore.USER_LOG_BUCKET,
temporary=True,
fail_fast=False))
# Cut the timeout allowed for flushing buffers on close to something small.
persistor._CLOSE_TIMEOUT_SECS = 0.100
for batch in batches:
persistor.PersistLogBatch(batch)
self._RunAsync(persistor.close)
self._VerifyBackupBatches(backup_dir, batches)
# Set a functional file object store instance and verify that it
# restores the pending server logs.
ObjectStore.SetInstance(ObjectStore.SERVER_LOG, oldStores[0])
ObjectStore.SetInstance(ObjectStore.USER_LOG, oldStores[1])
persistor = LogBatchPersistor(backup_dir=backup_dir)
self._RunAsync(persistor.Wait)
self._RunAsync(self._VerifyObjStoreBatches, batches)