def test_successful_run(logminer_contents_query_results, expected_record_count,
expected_record_types, mocker, setup):
(extractor, mockdb, mockargv, mock_producer, start_scn, end_scn) = setup
logminer_results_deque = deque(logminer_contents_query_results)
def execute_query_se(query, arraysize):
if STARTOF_MINMAX_QUERY in query:
mock_query_results_config = {'get_col_index.return_value': 1}
mock_query_results = Mock(**mock_query_results_config)
mock_query_results.__iter__ = Mock(return_value=iter([[1, 2]]))
elif "FROM v$logmnr_contents" in query:
mock_query_results_config = {
'get_col_index.return_value':
1,
'fetchone.return_value':
(None if not logminer_results_deque else
logminer_results_deque.popleft())
}
mock_query_results = Mock(**mock_query_results_config)
# Mock a single result with two columns returned
mock_query_results.__iter__ = Mock(
return_value=iter(logminer_results_deque))
else:
mock_query_results_config = {
'fetchall.return_value': [('schema', 'table')]
}
mock_query_results = Mock(**mock_query_results_config)
mock_query_results.__iter__ = Mock(return_value=iter([]))
return mock_query_results
mockdb.execute_query.side_effect = execute_query_se
expected_record_types_deque = deque(expected_record_types)
def write_se(message):
if message['record_type'] == const.END_OF_BATCH:
assert message['record_count'] == expected_record_count
assert len(expected_record_types_deque) != 0
rt = expected_record_types_deque.popleft()
print("actual={}, expected={}".format(message['record_type'], rt))
assert message['record_type'] == rt
return True
mock_producer.write.side_effect = write_se
unittest_utils.mock_get_schemas_and_tables(mocker, ['MYSCHEMA'],
['MYTABLE'])
extractor.extract()
if len(expected_record_types_deque) > 0:
pytest.fail(
"The remaining message record types were not sent: {}".format(
expected_record_types_deque))
def test_extract_no_schemas_or_tables(mocker, setup):
(extractor, mockdb, mockargv, mock_producer, start_scn, end_scn) = setup
unittest_utils.mock_get_schemas_and_tables(mocker, list(), list())
with pytest.raises(InvalidArgumentsError) as invalid_args_error:
extractor.extract()
expect_errstr = ("Both schemas ([]) and tables ([]) must be defined "
"in source_system_profile for profile 'myprofile'")
assert expect_errstr in str(invalid_args_error.value)
def test_poll_cdcs_valid_start_end_scn(mocker, setup):
(extractor, mockdb, mockargv, mock_producer, start_scn, end_scn) = setup
unittest_utils.mock_get_schemas_and_tables(mocker, [], [])
extractor.poll_cdcs(start_scn, end_scn)
mockdb.assert_has_calls([
mocker.call.execute(const.ALTER_NLS_DATE_FORMAT_COMMAND),
mocker.call.execute(const.ALTER_NLS_TIMESTAMP_FORMAT_COMMAND),
mocker.call.execute_stored_proc(
EXPECT_START_LOGMNR_CMD.format(
start_scn, end_scn,
build_logminer_options(BASE_LOGMNR_OPTIONS + ONLINE_DICT))),
mocker.call.execute_query(mocker.ANY, 1000),
mocker.call.execute_stored_proc(STOP_LOGMINER_COMMAND)
])
def test_no_redolog_dict(mocker, setup_no_redolog_dict):
(extractor, mockdb, mockargv, mock_producer, start_scn,
end_scn) = setup_no_redolog_dict
unittest_utils.mock_get_schemas_and_tables(mocker, [], [])
extractor.poll_cdcs(start_scn, end_scn)
mockdb.assert_has_calls([
mocker.call.execute(const.ALTER_NLS_DATE_FORMAT_COMMAND),
mocker.call.execute(const.ALTER_NLS_TIMESTAMP_FORMAT_COMMAND),
mocker.call.execute_query(GET_DICTIONARY_QUERY.format(scn=start_scn),
1000),
# If no redolog is found, then should go into online dict mode
mocker.call.execute_stored_proc(
EXPECT_START_LOGMNR_CMD.format(
start_scn, end_scn,
build_logminer_options(BASE_LOGMNR_OPTIONS + ONLINE_DICT))),
mocker.call.execute_query(mocker.ANY, 1000),
mocker.call.execute_stored_proc(STOP_LOGMINER_COMMAND)
])
def test_redolog_dict(mocker, setup_redolog_dict):
(extractor, mockdb, mockargv, mock_producer, start_scn,
end_scn) = setup_redolog_dict
unittest_utils.mock_get_schemas_and_tables(mocker, [], [])
extractor.poll_cdcs(start_scn, end_scn)
mockdb.assert_has_calls([
mocker.call.execute(const.ALTER_NLS_DATE_FORMAT_COMMAND),
mocker.call.execute(const.ALTER_NLS_TIMESTAMP_FORMAT_COMMAND),
mocker.call.execute_query(GET_DICTIONARY_QUERY.format(scn=start_scn),
1000),
mocker.call.execute_stored_proc(
"DBMS_LOGMNR.ADD_LOGFILE(LOGFILENAME => 'my_redolog_name', OPTIONS => DBMS_LOGMNR.NEW)"
),
mocker.call.execute_stored_proc(
EXPECT_START_LOGMNR_CMD.format(
start_scn, end_scn,
build_logminer_options(BASE_LOGMNR_OPTIONS + REDOLOG_DICT))),
mocker.call.execute_query(mocker.ANY, 1000),
mocker.call.execute_stored_proc(STOP_LOGMINER_COMMAND)
])
def setup_scn_override(tmpdir, mocker):
ARGV_STARTSCN = 99
ARGV_ENDSCN = 100
mockargv_config = unittest_utils.get_default_argv_config(tmpdir)
mockargv_config = utils.merge_dicts(mockargv_config, {
'startscn': ARGV_STARTSCN,
'endscn': ARGV_ENDSCN,
})
mockargv = mocker.Mock(**mockargv_config)
unittest_utils.setup_logging(mockargv.workdirectory)
unittest_utils.mock_get_schemas_and_tables(mocker, [], [])
db_config = {'execute_query.side_effect': default_execute_query_se}
mockdb = mocker.Mock(**db_config)
mock_audit_factory = unittest_utils.build_mock_audit_factory(mocker)
unittest_utils.mock_get_prev_run_cdcs(mocker)
mock_producer = unittest_utils.mock_build_kafka_producer(mocker)
yield (OracleCdcExtractor(mockdb, mockargv, mock_audit_factory), mockdb,
mockargv, ARGV_STARTSCN, ARGV_ENDSCN)
def test_poll_cdcs_start_gt_end_scn(mocker, setup):
(extractor, mockdb, mockargv, mock_producer, start_scn, end_scn) = setup
unittest_utils.mock_get_schemas_and_tables(mocker, [], [])
with pytest.raises(InvalidArgumentsError):
extractor.poll_cdcs(2, 1)