コード例 #1
0
 def _test_common_body(self, query_list, expected_data_list, collect_task_list):
     self._prepare_source_data()
     self._create_logger_expectations(query_list, expected_data_list)
     self._mox.ReplayAll()
     source_connection_factory = lambda: psycopg2.connect(self._source_db_manager.connection_string)
     dest_storage = SqliteStorage(self._dest_db_manager.connection_string, self._dest_storage_logger)
     collect_task = DbSourceCollectTask(self._source_id, collect_task_list, source_connection_factory, dest_storage, self._main_logger)
     result = collect_task.execute()
     self.assertTrue(result)
     actual = self._dest_db_manager.execute_query('select ID, SOURCE, CATEGORY, TIMEMARKER, DATA from STAT_DATA')
     self._check_data(self._source_id, expected_data_list, actual)
     self._mox.VerifyAll()
コード例 #2
0
def execute():
    source_id = 'ics.db'
    # logger
    logging.config.dictConfig(settings.LOG_CONF)
    root_logger = logging.getLogger('stat_ics_db_collector.entry_point')
    # source
    ics_db_conn_str = settings.ICS_DB_CONN_STR
    prepared_conn_str = dict(map(lambda k: (k, str(ics_db_conn_str[k])), ics_db_conn_str))
    source_connect_factory = lambda: kinterbasdb.connect(**prepared_conn_str)
    # dest
    dest_db_conn_str = settings.DEST_DB_CONN_STR
    dest_storage = SqliteStorage(dest_db_conn_str, root_logger.getChild('dest_storage'))
    # collect_task
    collect_task_list = collect_task_def.collect_task_def
    # db source collect task
    task = DbSourceCollectTask(source_id, collect_task_list, source_connect_factory, dest_storage, root_logger)
    return task.execute()
コード例 #3
0
 def setUp(self):
     self._mox = Mox()
     self._source_connection = self._mox.CreateMockAnything()
     self._source_cursor = self._mox.CreateMockAnything()
     self._dest_storage = self._mox.CreateMockAnything()
     self._logger = self._mox.CreateMock(Logger)
     self._db_source_collector_logger = self._mox.CreateMock(Logger)
     self._data_collector_logger = self._mox.CreateMock(Logger)
     self._collect_task_logger = self._mox.CreateMock(Logger)
     self._dest_storage_logger = self._mox.CreateMock(Logger)
     category_transformer = lambda row: 'category.{0!s}'.format(row[0])
     data_transformer = lambda row: '{0!s}:{1!s}'.format(row[1], row[2])
     self._collect_task_list = [CollectTask(['q1', 'q2'], SimpleProcessTask()),
                                CollectTask(['q3'], TransformProcessTask(category_transformer, data_transformer))]
     source_conn_factory = lambda: self._source_connection
     self._collect_task = DbSourceCollectTask(self._source_id, self._collect_task_list, source_conn_factory, self._dest_storage, self._logger)
コード例 #4
0
class TestDbSourceCollectTask(TestCase):
    def __init__(self, methodName='runTest'):
        super(TestDbSourceCollectTask, self).__init__(methodName)
        self._mox = None
        self._source_connection = None
        self._source_cursor = None
        self._dest_storage = None
        self._logger = None
        self._data_collector_logger = None
        self._collect_task_logger = None
        self._source_id = 'test_db_source'
        self._collect_task_list = None
        self._collector = None

    def setUp(self):
        self._mox = Mox()
        self._source_connection = self._mox.CreateMockAnything()
        self._source_cursor = self._mox.CreateMockAnything()
        self._dest_storage = self._mox.CreateMockAnything()
        self._logger = self._mox.CreateMock(Logger)
        self._db_source_collector_logger = self._mox.CreateMock(Logger)
        self._data_collector_logger = self._mox.CreateMock(Logger)
        self._collect_task_logger = self._mox.CreateMock(Logger)
        self._dest_storage_logger = self._mox.CreateMock(Logger)
        category_transformer = lambda row: 'category.{0!s}'.format(row[0])
        data_transformer = lambda row: '{0!s}:{1!s}'.format(row[1], row[2])
        self._collect_task_list = [CollectTask(['q1', 'q2'], SimpleProcessTask()),
                                   CollectTask(['q3'], TransformProcessTask(category_transformer, data_transformer))]
        source_conn_factory = lambda: self._source_connection
        self._collect_task = DbSourceCollectTask(self._source_id, self._collect_task_list, source_conn_factory, self._dest_storage, self._logger)

    def test_normal_collect(self):
        self._logger.info('execute() enter')
        self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
        self._db_source_collector_logger.info('collect() enter')
        self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
        self._data_collector_logger.info('collect_data(collect_task_list) enter')
        self._source_connection.cursor().AndReturn(self._source_cursor)
        self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        set_collect_data_expectations(self._collect_task_logger,
            self._source_cursor,
            ['q1', 'q2'],
            [[('cat1', 'data1'), ('cat2', 'data2')], [('cat1', 'data3')]])
        set_collect_data_expectations(self._collect_task_logger,
            self._source_cursor,
            ['q3'],
            [[('cat1', 'data1', 'subdata1'), ('cat2', 'data2', 'subdata2'), ('cat1', 'data3', 'subdata3')]])
        self._source_connection.close()
        self._data_collector_logger.info(u'collect_data(collect_task_list) exit')
        self._db_source_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        save_items_list = [[DataItem(category='cat1', data='data1'), DataItem(category='cat2', data='data2'), DataItem(category='cat1', data='data3')],
            [DataItem(category='category.cat1', data='data1:subdata1'), DataItem(category='category.cat2', data='data2:subdata2'), DataItem(category='category.cat1', data='data3:subdata3')]]
        set_process_data_expectations(self._collect_task_logger, self._dest_storage, self._source_id, save_items_list)
        self._db_source_collector_logger.info('collect() exit')
        self._logger.info('execute() exit')
        self._mox.ReplayAll()
        self.assertTrue(self._collect_task.execute())
        self._mox.VerifyAll()

    def test_exception_when_collect(self):
        self._logger.info('execute() enter')
        self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
        self._db_source_collector_logger.info('collect() enter')
        self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
        self._data_collector_logger.info('collect_data(collect_task_list) enter')
        self._source_connection.cursor().AndReturn(self._source_cursor)
        self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        self._collect_task_logger.info('collect_data(query_executer) enter')
        self._collect_task_logger.info('_collect_data_item(q1) enter')
        self._source_cursor.execute('q1').AndRaise(CustomTestException())
        self._collect_task_logger.exception('exception in _collect_data_item(q1)')
        self._collect_task_logger.exception('exception in collect_data(query_executer)')
        self._source_connection.close()
        self._data_collector_logger.exception('exception in collect_data(collect_task_list)')
        self._db_source_collector_logger.exception('exception in collect()')
        self._logger.exception('exception in execute()')
        self._mox.ReplayAll()
        self.assertFalse(self._collect_task.execute())
        self._mox.VerifyAll()

    def test_exception_when_process(self):
        self._logger.info('execute() enter')
        self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
        self._db_source_collector_logger.info('collect() enter')
        self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
        self._data_collector_logger.info('collect_data(collect_task_list) enter')
        self._source_connection.cursor().AndReturn(self._source_cursor)
        self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        set_collect_data_expectations(self._collect_task_logger,
            self._source_cursor,
            ['q1', 'q2'],
            [[('cat1',), ('cat2',)], [('cat1',)]])
        set_collect_data_expectations(self._collect_task_logger,
            self._source_cursor,
            ['q3'],
            [[('cat1', 'data1', 'subdata1'), ('cat2', 'data2', 'subdata2'), ('cat1', 'data3', 'subdata3')]])
        self._source_connection.close()
        self._data_collector_logger.info(u'collect_data(collect_task_list) exit')
        self._db_source_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        self._collect_task_logger.info('process_data() enter')
        self._collect_task_logger.exception('exception in process_data()')
        self._db_source_collector_logger.exception(u'exception in collect()')
        self._logger.exception('exception in execute()')
        self._mox.ReplayAll()
        self.assertFalse(self._collect_task.execute())
        self._mox.VerifyAll()

    def test_exception_when_save(self):
        self._logger.info('execute() enter')
        self._logger.getChild('db_source_collector').AndReturn(self._db_source_collector_logger)
        self._db_source_collector_logger.info('collect() enter')
        self._db_source_collector_logger.getChild('data_collector').AndReturn(self._data_collector_logger)
        self._data_collector_logger.info('collect_data(collect_task_list) enter')
        self._source_connection.cursor().AndReturn(self._source_cursor)
        self._data_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        set_collect_data_expectations(self._collect_task_logger,
            self._source_cursor,
            ['q1', 'q2'],
            [[('cat1', 'data1'), ('cat2', 'data2')], [('cat1', 'data3')]])
        set_collect_data_expectations(self._collect_task_logger,
            self._source_cursor,
            ['q3'],
            [[('cat1', 'data1', 'subdata1'), ('cat2', 'data2', 'subdata2'), ('cat1', 'data3', 'subdata3')]])
        self._source_connection.close()
        self._data_collector_logger.info(u'collect_data(collect_task_list) exit')
        self._db_source_collector_logger.getChild('collect_task').AndReturn(self._collect_task_logger)
        self._collect_task_logger.info('process_data() enter')
        self._collect_task_logger.info('process_data() exit')
        save_data = [DataItem(category='cat1', data='data1'), DataItem(category='cat2', data='data2'), DataItem(category='cat1', data='data3')]
        self._dest_storage.save_data(self._source_id, save_data).AndRaise(CustomTestException())
        self._db_source_collector_logger.exception(u'exception in collect()')
        self._logger.exception('exception in execute()')
        self._mox.ReplayAll()
        self.assertFalse(self._collect_task.execute())
        self._mox.VerifyAll()