def test_initsync_table(mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
"sourcedbtype": const.ORACLE,
"targetdbtype": const.POSTGRES,
})
mockargv = mocker.Mock(**mockargv_config)
source_conn_detail = dbuser.get_dbuser_properties(mockargv.sourceuser)
target_conn_detail = dbuser.get_dbuser_properties(mockargv.targetuser)
source_schema = "myschema"
target_schema = "myschema"
process_control_id = 1
query_condition = None
mock_msg_queue_config = {"put.return_value": None}
mock_msg_queue = mocker.Mock(**mock_msg_queue_config)
mockdb = build_mockdb(mocker, db_config, const.ORACLE)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
mock_update_ssp = mocker.patch("data_pipeline.initsync_pipe._update_source_system_profile")
# We don't want a real process being forked during tests
mock_process = mocker.patch("data_pipeline.initsync_pipe.Process")
initsync_pipe.initsync_table(mockargv, source_conn_detail,
target_conn_detail, source_schema, table.name, target_schema,
process_control_id, query_condition, mock_msg_queue)
assert mock_update_ssp.call_count == 2
def setup_no_redolog_dict(tmpdir, mocker):
mockargv_config = unittest_utils.get_default_argv_config(tmpdir)
mockargv_config = utils.merge_dicts(
mockargv_config, {
'donotload': False,
'donotsend': False,
'streamhost': 'host',
'streamchannel': 'channel',
'streamschemafile': 'file',
'streamschemahost': 'host',
'sourcedictionary': 'redolog'
})
mockargv = mocker.Mock(**mockargv_config)
unittest_utils.setup_logging(mockargv.workdirectory)
db_config = {'execute_query.side_effect': execute_query_no_redolog_se}
mockdb = mocker.Mock(**db_config)
mock_audit_factory = unittest_utils.build_mock_audit_factory(mocker)
unittest_utils.mock_get_prev_run_cdcs(mocker)
mock_producer = unittest_utils.mock_build_kafka_producer(mocker)
yield (OracleCdcExtractor(mockdb, mockargv, mock_audit_factory), mockdb,
mockargv, mock_producer, 1, 2)
def test_extract_data(data_extract_data, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
column_list = data_extract_data.input_column_list
dbtype = data_extract_data.input_dbtype
extractlsn = data_extract_data.input_extractlsn
samplerows = data_extract_data.input_samplerows
lock = data_extract_data.input_lock
query_condition = data_extract_data.input_query_condition
expected_sql = data_extract_data.expected_sql
metacols= data_extract_data.input_metacols
mockargv_config = utils.merge_dicts(mockargv_config, {
'lock': lock,
'samplerows': samplerows,
'extractlsn': extractlsn,
'metacols': metacols,
})
mockargv = mocker.Mock(**mockargv_config)
mockdb = build_mockdb(mocker, db_config, dbtype)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
def mock_function(argv, table, extract_data_sql):
pass
db.extract_data(column_list, table, query_condition, mock_function)
# Assert called with SQL str
mockdb.execute_query.assert_called_once_with(
expected_sql, const.DEFAULT_ARRAYSIZE,
post_process_func=mocker.ANY)
def setup(tmpdir, mocker):
mockargv_config = unittest_utils.get_default_argv_config(tmpdir)
mockargv_config = utils.merge_dicts(mockargv_config, {
'auditschema': TEST_AUDIT_SCHEMA
})
mockargv = mocker.Mock(**mockargv_config)
unittest_utils.setup_logging(mockargv.workdirectory)
mock_get_program_args = mocker.patch(
'data_pipeline.initsync_pipe.get_program_args')
mock_get_program_args.return_value = mockargv
db_config = {
'execute_query.side_effect': default_execute_query_se,
'close.return_value': None,
}
logger = logging.getLogger(__name__)
table = sql_utils.TableName('myschema', 'mytable')
mock_process_control_constructor(mocker)
mock_process_control_detail_constructor(mocker)
yield (mockargv_config, db_config, table, logger, tmpdir)
def test_apply(nullstring, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
"nullstring": nullstring,
})
mockargv = mocker.Mock(**mockargv_config)
db_config = utils.merge_dicts(db_config, {
"copy_expert.return_value": 99,
})
mockdb = build_mockdb(mocker, db_config, const.POSTGRES)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
mock_msg_queue_config = {"put.return_value": None}
mock_msg_queue = mocker.Mock(**mock_msg_queue_config)
pipe_file = os.path.join(mockargv.workdirectory, "myfakefifo")
open(pipe_file, 'a').close()
mock_pc_detail_config = {"update.return_value":None}
mock_pc_detail = mocker.Mock(**mock_pc_detail_config)
mock_pc_detail_cons = mocker.patch(
"data_pipeline.initsync_pipe.ProcessControlDetail")
mock_pc_detail_cons.return_value = mock_pc_detail
initsync_pipe.apply(mockargv, pipe_file, table, ["col1", "col2"],
db, 1, mock_msg_queue)
mockdb.copy_expert.assert_called_once_with(
input_file=mocker.ANY,
table_name="myschema.mytable",
sep=const.FIELD_DELIMITER,
null_string=nullstring,
column_list=['col1', 'col2', 'ctl_ins_ts', 'ctl_upd_ts'],
quote_char=chr(const.ASCII_GROUPSEPARATOR),
escape_char=chr(const.ASCII_RECORDSEPARATOR),
size=mockargv.buffersize
)
mock_pc_detail.update.assert_called_once_with(
comment=mocker.ANY,
status=const.SUCCESS,
source_row_count=99,
insert_row_count=99,
)
def test_extract(raise_exception, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
"samplerows": 10,
})
mockargv = mocker.Mock(**mockargv_config)
db_config = utils.merge_dicts(db_config, {
"copy_expert.return_value": 99,
})
mock_extract_data = mocker.patch(
"data_pipeline.initsync_pipe.ProcessControlDetail")
# if raise_exception:
# raise Exception("Failed to run mock_function")
mockdb = build_mockdb(mocker, db_config, const.POSTGRES)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
pipe_file = os.path.join(mockargv.workdirectory, "myfakefifo")
open(pipe_file, 'a').close()
mock_pc_detail_config = {"update.return_value":None}
mock_pc_detail = mocker.Mock(**mock_pc_detail_config)
mock_pc_detail_cons = mocker.patch(
"data_pipeline.initsync_pipe.ProcessControlDetail")
mock_pc_detail_cons.return_value = mock_pc_detail
mock_msg_queue_config = {"put.return_value": None}
mock_msg_queue = mocker.Mock(**mock_msg_queue_config)
initsync_pipe.extract(mockargv, pipe_file, table, ["col1", "col2"],
db, 1, None, mock_msg_queue)
mockdb.execute_query.assert_called_once_with("""
SELECT
col1
, col2
, CURRENT_TIMESTAMP AS INS_TIMESTAMP -- insert timestamp
, CURRENT_TIMESTAMP AS UPD_TIMESTAMP -- update timestamp
, 'UNSUPPORTED BY DB'
FROM myschema.mytable
WHERE 1=1
LIMIT 10""", 1000, post_process_func=mocker.ANY)
def test_get_source_db(dbtype, mocker, setup):
(mockargv_config) = setup
mockargv_config = utils.merge_dicts(mockargv_config,
{"sourcedbtype": dbtype})
mockargv = mocker.Mock(**mockargv_config)
db = extract.get_source_db(mockargv)
assert type(db).__name__.lower() == "{}db".format(dbtype.lower())
def test_delete_truncate(query_condition, delete, truncate, expect_sql, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
'delete': delete,
'truncate': truncate
})
mockargv = mocker.Mock(**mockargv_config)
mockdb = build_mockdb(mocker, db_config, const.POSTGRES)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
initsync_pipe.clean_target_table(mockargv, db, table, query_condition, 1)
mockdb.execute.assert_called_once_with(expect_sql)
def test_write(nullstring, record, expected_payload, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
'nullstring': nullstring,
})
mockargv = mocker.Mock(**mockargv_config)
mock_fifo_file = mocker.Mock(**{"write.return_value": None})
lsn = initsync_pipe.write(mockargv, record, mock_fifo_file, None)
mock_fifo_file.write.assert_called_once_with(expected_payload)
assert lsn == 99
def test_analyze_enabled(mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
'vacuum': False,
'analyze': True
})
mockargv = mocker.Mock(**mockargv_config)
mockdb = build_mockdb(mocker, db_config, const.POSTGRES)
db = build_initsyncdb(mocker, mockdb, mockargv, logger)
table_name = sql_utils.TableName("ctl", "mytable")
initsync_pipe.execute_post_processing(table_name, db)
mockdb.execute.assert_called_with("ANALYZE ctl.mytable")
def test_main(ssps, streamhost, streamchannel, seektoend_called_times, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
mockargv_config = utils.merge_dicts(mockargv_config, {
'streamhost': streamhost,
'streamchannel': streamchannel,
})
mockargv = mocker.Mock(**mockargv_config)
def seek_to_end_se(timeout):
print("kafka_consumer: seek_to_end called with timeout = {}"
.format(timeout))
mock_kafka_consumer_config = { "seek_to_end.side_effect": seek_to_end_se }
mock_kafka_consumer = mocker.Mock(**mock_kafka_consumer_config)
mock_get_program_args = mocker.patch(
"data_pipeline.initsync_pipe.get_program_args")
mock_get_program_args.return_value = mockargv
mock_kafka_consumer_cons = mocker.patch(
"data_pipeline.stream.factory.KafkaConsumer")
mock_kafka_consumer_cons.return_value = mock_kafka_consumer
mock_get_ssp = mocker.patch(
"data_pipeline.initsync_pipe.get_source_system_profile_params")
mock_get_ssp.return_value = ssps
mock_parallelise_initsync = mocker.patch(
"data_pipeline.initsync_pipe.parallelise_initsync")
mock_parallelise_initsync.return_value = {
table: ("LSN0", const.SUCCESS, "CCPI", "All good")
}
def report_error_se(message, process_control, logger):
print("An error occurred in test: {}".format(message))
mock_report_error = mocker.patch("data_pipeline.initsync_pipe.report_error")
mock_report_error.size_effect = report_error_se
mock_send = mocker.patch("data_pipeline.initsync_pipe.mailer.send")
initsync_pipe.main()
assert mock_kafka_consumer.seek_to_end.call_count == seektoend_called_times
mock_report_error.assert_not_called()
mock_send.assert_called_once()
def setup_kill(tmpdir, mocker):
mockargv_config = unittest_utils.get_default_argv_config(tmpdir)
mockargv_config = utils.merge_dicts(mockargv_config, {
'donotload': False,
'donotsend': False,
'kill': True
})
mockargv = mocker.Mock(**mockargv_config)
unittest_utils.setup_logging(mockargv.workdirectory)
db_config = {}
mockdb = mocker.Mock(**db_config)
mock_audit_factory = unittest_utils.build_mock_audit_factory(mocker)
unittest_utils.mock_get_prev_run_cdcs(mocker)
mock_producer = unittest_utils.mock_build_kafka_producer(mocker)
yield (OracleCdcExtractor(mockdb, mockargv, mock_audit_factory), mockdb,
mockargv, mock_producer)
def test_get_source_system_profile_params_without_auditdb(
tablelist, isfile, expected_ssp_params, mocker, setup):
(mockargv_config, db_config, table, logger, tmpdir) = setup
# Create a mock tablelist file
if isfile:
filename = str(tmpdir.mkdir("ssp").join(tablelist[0]))
with open(filename, 'a') as f:
f.write("tableA\n")
f.write("tableB\n")
f.write("tableC\n")
tablelist = [filename]
mockargv_config = utils.merge_dicts(mockargv_config, {
'auditschema': None,
'audituser': None,
'tablelist': tablelist,
})
mockargv = mocker.Mock(**mockargv_config)
ssp_params = initsync_pipe.get_source_system_profile_params(mockargv)
assert ssp_params == expected_ssp_params
def setup_scn_override(tmpdir, mocker):
ARGV_STARTSCN = 99
ARGV_ENDSCN = 100
mockargv_config = unittest_utils.get_default_argv_config(tmpdir)
mockargv_config = utils.merge_dicts(mockargv_config, {
'startscn': ARGV_STARTSCN,
'endscn': ARGV_ENDSCN,
})
mockargv = mocker.Mock(**mockargv_config)
unittest_utils.setup_logging(mockargv.workdirectory)
unittest_utils.mock_get_schemas_and_tables(mocker, [], [])
db_config = {'execute_query.side_effect': default_execute_query_se}
mockdb = mocker.Mock(**db_config)
mock_audit_factory = unittest_utils.build_mock_audit_factory(mocker)
unittest_utils.mock_get_prev_run_cdcs(mocker)
mock_producer = unittest_utils.mock_build_kafka_producer(mocker)
yield (OracleCdcExtractor(mockdb, mockargv, mock_audit_factory), mockdb,
mockargv, ARGV_STARTSCN, ARGV_ENDSCN)
def setup_dependencies(tmpdir, mocker, metacols, inactive_applied_tables,
config_to_merge):
if not inactive_applied_tables:
inactive_applied_tables = set()
mockdbstate = MockDbState()
db_config = {
'closed.side_effect': mockdbstate.closed,
'connect.side_effect': mockdbstate.connect,
'close.side_effect': mockdbstate.close
}
mock_target_db = mocker.Mock(**db_config)
p = tmpdir.mkdir("test_apply_sql_dir")
mockargv_config = unittest_utils.get_default_argv_config(tmpdir)
mockargv_config = utils.merge_dicts(mockargv_config, config_to_merge)
mockargv = mocker.Mock(**mockargv_config)
unittest_utils.mock_get_program_args(
mocker, 'data_pipeline.applier.applier.get_program_args', mockargv)
unittest_utils.setup_logging(mockargv.workdirectory)
unittest_utils.mock_get_inactive_applied_tables(mocker,
inactive_applied_tables)
mock_audit_factory = build_mock_audit_factory(mocker, mockargv)
mock_audit_db = patch_db_factory(mocker)
oracle_processor = OracleCdcProcessor(metacols)
f = mocker.patch.object(Applier, '_get_max_lsn_source_system_profile')
return (oracle_processor, mock_target_db, mockargv, mock_audit_factory,
mock_audit_db)
def build_mockdb(mocker, base_config, dbtype):
db_config = utils.merge_dicts(base_config, {
'dbtype': dbtype
})
return mocker.Mock(**db_config)
