def test_cdc_base_plugin_execute(self):
cdc_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_names": []
}
with self.assertRaises(NotImplementedError) as cm:
cdc_base_object = CDCBase(**cdc_init_params)
cdc_base_object.execute()
def setUpClass(cls) -> None:
logging.disable(logging.CRITICAL)
cls.postgres_testing = testing.postgresql.PostgresqlFactory(cache_initialized_db=True)
# setup postgres db
cls.setup_postgres()
# load all derived classes for the test setup
CDCBase.load_derived_classes()
FullLoadBase.load_derived_classes()
BaseDataBaseOperator.load_derived_classes()
def test_cdc_base_plugin_empty_table(self):
cdc_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_names": None
}
with self.assertRaises(ValueError) as cm:
CDCBase(**cdc_init_params)
self.assertEqual(str(cm.exception), "Table name is None or Empty")
cdc_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_names": "public.employee"
}
with self.assertRaises(ValueError) as cm:
CDCBase(**cdc_init_params)
self.assertEqual(str(cm.exception), "Table name is None or Empty")
def test_cdc_base_plugin_empty_output(self):
cdc_init_params = {
"output_folder_location": None,
"connection_string": self.postgres_connection_string,
"table_names": []
}
with self.assertRaises(ValueError) as cm:
CDCBase(**cdc_init_params)
self.assertEqual(str(cm.exception), "output_folder_location is empty")
def initialize():
""" Initialize the lineage settings """
# register interrupt signal
register_ctrl_c_signal()
# create the logger
create_rotating_log()
database_operator_name = configuration.get("conf", "database_operator")
database_operator_type = configuration.get("conf",
"database_operator_type")
connection_string = configuration.get("conf", "connection_string")
load_type = configuration.get("conf", "load_type")
full_load_plugin_name = configuration.get("conf", "full_load_plugin_name")
cdc_plugin_name = configuration.get("conf", "cdc_plugin_name")
table_names_str = configuration.get("conf", "table_names", default="")
table_names = table_names_str.split(
",") if table_names_str.strip() != "" else []
output_location = configuration.get("conf", "output_location")
BaseDataBaseOperator.load_derived_classes()
FullLoadBase.load_derived_classes()
CDCBase.load_derived_classes()
print(database_operator_name, database_operator_type, load_type,
full_load_plugin_name, cdc_plugin_name, table_names_str, table_names,
output_location)
database_operator = BaseDataBaseOperator.get_object(
database_operator_type, database_operator_name)
database_operator_params = {
"connection_string": connection_string,
"load_type": LoadType[load_type],
"table_names": table_names,
"full_load_plugin_name": full_load_plugin_name,
"cdc_plugin_name": cdc_plugin_name,
"output_location": output_location,
}
retry_times = int(configuration.get("conf", "retry_times", 0))
run_database_operator(database_operator, database_operator_params,
retry_times)
def test_cdc_base_plugin_parameters(self):
cdc_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_names": ["public.employee"]
}
cdc_object = CDCBase(**cdc_init_params)
self.assertEqual(
list(cdc_object.__dict__.keys()).sort(), [
"connection_string", "output_folder_location", "table_names",
"configuration", "logger"
].sort())
def _run_cdc_process(cdc_plugin_name: str,
cdc_init_params: Dict) -> None:
"""
Worker process for cdc process to complete
:param cdc_plugin_name: plugin to be used
:type cdc_plugin_name: str
:param cdc_init_params:
:type cdc_init_params:
"""
cdc_plugin = CDCBase.get_object(cdc_plugin_name)
plugin_parameters = cdc_plugin.plugin_parameters
cdc_init_params = \
PostgresOperator.append_plugin_parameter_from_configuration(
cdc_init_params,
plugin_parameters
)
create_rotating_log("cdc", "siirto.log", "siirto")
cdc_object = cdc_plugin(**cdc_init_params)
cdc_object.setup_graceful_shutdown()
cdc_object.execute()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.full_load_plugin = FullLoadBase.get_object(self.full_load_plugin_name)
self.cdc_plugin = CDCBase.get_object(self.cdc_plugin_name)
self._validate_input_parameters()
def test_cdc_base_plugin_load_cdc_default_plugin(self):
CDCBase.load_derived_classes()
child_operator_object = CDCBase.get_object("PgDefaultCDCPlugin")
self.assertEqual(child_operator_object.__dict__["__module__"],
"siirto.plugins.cdc.pg_default_cdc_plugin")
self.assertEqual(child_operator_object.__name__, "PgDefaultCDCPlugin")