def _test_common_body(self, query_list, expected_data_list, collect_task_list):
self._prepare_source_data()
self._create_logger_expectations(query_list, expected_data_list)
self._mox.ReplayAll()
source_connection_factory = lambda: psycopg2.connect(self._source_db_manager.connection_string)
dest_storage = SqliteStorage(self._dest_db_manager.connection_string, self._dest_storage_logger)
collect_task = DbSourceCollectTask(self._source_id, collect_task_list, source_connection_factory, dest_storage, self._main_logger)
result = collect_task.execute()
self.assertTrue(result)
actual = self._dest_db_manager.execute_query('select ID, SOURCE, CATEGORY, TIMEMARKER, DATA from STAT_DATA')
self._check_data(self._source_id, expected_data_list, actual)
self._mox.VerifyAll()
def execute():
source_id = 'ics.db'
# logger
logging.config.dictConfig(settings.LOG_CONF)
root_logger = logging.getLogger('stat_ics_db_collector.entry_point')
# source
ics_db_conn_str = settings.ICS_DB_CONN_STR
prepared_conn_str = dict(map(lambda k: (k, str(ics_db_conn_str[k])), ics_db_conn_str))
source_connect_factory = lambda: kinterbasdb.connect(**prepared_conn_str)
# dest
dest_db_conn_str = settings.DEST_DB_CONN_STR
dest_storage = SqliteStorage(dest_db_conn_str, root_logger.getChild('dest_storage'))
# collect_task
collect_task_list = collect_task_def.collect_task_def
# db source collect task
task = DbSourceCollectTask(source_id, collect_task_list, source_connect_factory, dest_storage, root_logger)
return task.execute()
def setUp(self):
self._mox = Mox()
self._source_connection = self._mox.CreateMockAnything()
self._source_cursor = self._mox.CreateMockAnything()
self._dest_storage = self._mox.CreateMockAnything()
self._logger = self._mox.CreateMock(Logger)
self._db_source_collector_logger = self._mox.CreateMock(Logger)
self._data_collector_logger = self._mox.CreateMock(Logger)
self._collect_task_logger = self._mox.CreateMock(Logger)
self._dest_storage_logger = self._mox.CreateMock(Logger)
category_transformer = lambda row: 'category.{0!s}'.format(row[0])
data_transformer = lambda row: '{0!s}:{1!s}'.format(row[1], row[2])
self._collect_task_list = [CollectTask(['q1', 'q2'], SimpleProcessTask()),
CollectTask(['q3'], TransformProcessTask(category_transformer, data_transformer))]
source_conn_factory = lambda: self._source_connection
self._collect_task = DbSourceCollectTask(self._source_id, self._collect_task_list, source_conn_factory, self._dest_storage, self._logger)
class TestDbSourceCollectTask(TestCase):
def __init__(self, methodName='runTest'):
super(TestDbSourceCollectTask, self).__init__(methodName)
self._mox = None
self._source_connection = None
self._source_cursor = None
self._dest_storage = None
self._logger = None
self._data_collector_logger = None
self._collect_task_logger = None
self._source_id = 'test_db_source'
self._collect_task_list = None
self._collector = None
def setUp(self):
self._mox = Mox()
self._source_connection = self._mox.CreateMockAnything()
self._source_cursor = self._mox.CreateMockAnything()
self._dest_storage = self._mox.CreateMockAnything()
self._logger = self._mox.CreateMock(Logger)
self._db_source_collector_logger = self._mox.CreateMock(Logger)
self._data_collector_logger = self._mox.CreateMock(Logger)
self._collect_task_logger = self._mox.CreateMock(Logger)
self._dest_storage_logger = self._mox.CreateMock(Logger)
category_transformer = lambda row: 'category.{0!s}'.format(row[0])
data_transformer = lambda row: '{0!s}:{1!s}'.format(row[1], row[2])
self._collect_task_list = [CollectTask(['q1', 'q2'], SimpleProcessTask()),
CollectTask(['q3'], TransformProcessTask(category_transformer, data_transformer))]
source_conn_factory = lambda: self._source_connection
self._collect_task = DbSourceCollectTask(self._source_id, self._collect_task_list, source_conn_factory, self._dest_storage, self._logger)
def test_normal_collect(self):
self._logger.info('execute() enter')
self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
self._db_source_collector_logger.info('collect() enter')
self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
self._data_collector_logger.info('collect_data(collect_task_list) enter')
self._source_connection.cursor().AndReturn(self._source_cursor)
self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
set_collect_data_expectations(self._collect_task_logger,
self._source_cursor,
['q1', 'q2'],
[[('cat1', 'data1'), ('cat2', 'data2')], [('cat1', 'data3')]])
set_collect_data_expectations(self._collect_task_logger,
self._source_cursor,
['q3'],
[[('cat1', 'data1', 'subdata1'), ('cat2', 'data2', 'subdata2'), ('cat1', 'data3', 'subdata3')]])
self._source_connection.close()
self._data_collector_logger.info(u'collect_data(collect_task_list) exit')
self._db_source_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
save_items_list = [[DataItem(category='cat1', data='data1'), DataItem(category='cat2', data='data2'), DataItem(category='cat1', data='data3')],
[DataItem(category='category.cat1', data='data1:subdata1'), DataItem(category='category.cat2', data='data2:subdata2'), DataItem(category='category.cat1', data='data3:subdata3')]]
set_process_data_expectations(self._collect_task_logger, self._dest_storage, self._source_id, save_items_list)
self._db_source_collector_logger.info('collect() exit')
self._logger.info('execute() exit')
self._mox.ReplayAll()
self.assertTrue(self._collect_task.execute())
self._mox.VerifyAll()
def test_exception_when_collect(self):
self._logger.info('execute() enter')
self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
self._db_source_collector_logger.info('collect() enter')
self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
self._data_collector_logger.info('collect_data(collect_task_list) enter')
self._source_connection.cursor().AndReturn(self._source_cursor)
self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
self._collect_task_logger.info('collect_data(query_executer) enter')
self._collect_task_logger.info('_collect_data_item(q1) enter')
self._source_cursor.execute('q1').AndRaise(CustomTestException())
self._collect_task_logger.exception('exception in _collect_data_item(q1)')
self._collect_task_logger.exception('exception in collect_data(query_executer)')
self._source_connection.close()
self._data_collector_logger.exception('exception in collect_data(collect_task_list)')
self._db_source_collector_logger.exception('exception in collect()')
self._logger.exception('exception in execute()')
self._mox.ReplayAll()
self.assertFalse(self._collect_task.execute())
self._mox.VerifyAll()
def test_exception_when_process(self):
self._logger.info('execute() enter')
self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
self._db_source_collector_logger.info('collect() enter')
self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
self._data_collector_logger.info('collect_data(collect_task_list) enter')
self._source_connection.cursor().AndReturn(self._source_cursor)
self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
set_collect_data_expectations(self._collect_task_logger,
self._source_cursor,
['q1', 'q2'],
[[('cat1',), ('cat2',)], [('cat1',)]])
set_collect_data_expectations(self._collect_task_logger,
self._source_cursor,
['q3'],
[[('cat1', 'data1', 'subdata1'), ('cat2', 'data2', 'subdata2'), ('cat1', 'data3', 'subdata3')]])
self._source_connection.close()
self._data_collector_logger.info(u'collect_data(collect_task_list) exit')
self._db_source_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
self._collect_task_logger.info('process_data() enter')
self._collect_task_logger.exception('exception in process_data()')
self._db_source_collector_logger.exception(u'exception in collect()')
self._logger.exception('exception in execute()')
self._mox.ReplayAll()
self.assertFalse(self._collect_task.execute())
self._mox.VerifyAll()
def test_exception_when_save(self):
self._logger.info('execute() enter')
self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
self._db_source_collector_logger.info('collect() enter')
self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
self._data_collector_logger.info('collect_data(collect_task_list) enter')
self._source_connection.cursor().AndReturn(self._source_cursor)
self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
set_collect_data_expectations(self._collect_task_logger,
self._source_cursor,
['q1', 'q2'],
[[('cat1', 'data1'), ('cat2', 'data2')], [('cat1', 'data3')]])
set_collect_data_expectations(self._collect_task_logger,
self._source_cursor,
['q3'],
[[('cat1', 'data1', 'subdata1'), ('cat2', 'data2', 'subdata2'), ('cat1', 'data3', 'subdata3')]])
self._source_connection.close()
self._data_collector_logger.info(u'collect_data(collect_task_list) exit')
self._db_source_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
self._collect_task_logger.info('process_data() enter')
self._collect_task_logger.info('process_data() exit')
save_data = [DataItem(category='cat1', data='data1'), DataItem(category='cat2', data='data2'), DataItem(category='cat1', data='data3')]
self._dest_storage.save_data(self._source_id, save_data).AndRaise(CustomTestException())
self._db_source_collector_logger.exception(u'exception in collect()')
self._logger.exception('exception in execute()')
self._mox.ReplayAll()
self.assertFalse(self._collect_task.execute())
self._mox.VerifyAll()