def setUpClass(cls) -> None:
logging.disable(logging.CRITICAL)
cls.postgres_testing = testing.postgresql.PostgresqlFactory(cache_initialized_db=True)
# setup postgres db
cls.setup_postgres()
# load all derived classes for the test setup
CDCBase.load_derived_classes()
FullLoadBase.load_derived_classes()
BaseDataBaseOperator.load_derived_classes()
def test_full_load_base_plugin_load_cdc_default_plugin(self):
FullLoadBase.load_derived_classes()
child_operator_object = FullLoadBase.get_object(
"PgDefaultFullLoadPlugin")
self.assertEqual(
child_operator_object.__dict__["__module__"],
"siirto.plugins.full_load.pg_default_full_load_plugin")
self.assertEqual(child_operator_object.__name__,
"PgDefaultFullLoadPlugin")
def test_full_load_base_plugin_execute(self):
full_load_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_name": "public.employee",
"notify_on_completion": None
}
with self.assertRaises(NotImplementedError) as cm:
full_load_base_object = FullLoadBase(**full_load_init_params)
full_load_base_object.execute()
def test_full_load_base_plugin_empty_table(self):
full_load_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_name": None,
"notify_on_completion": None
}
with self.assertRaises(ValueError) as cm:
FullLoadBase(**full_load_init_params)
self.assertEqual(str(cm.exception), "Table name is None")
full_load_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_name": "",
"notify_on_completion": None
}
with self.assertRaises(ValueError) as cm:
FullLoadBase(**full_load_init_params)
self.assertEqual(str(cm.exception), "Table name is None")
def initialize():
""" Initialize the lineage settings """
# register interrupt signal
register_ctrl_c_signal()
# create the logger
create_rotating_log()
database_operator_name = configuration.get("conf", "database_operator")
database_operator_type = configuration.get("conf",
"database_operator_type")
connection_string = configuration.get("conf", "connection_string")
load_type = configuration.get("conf", "load_type")
full_load_plugin_name = configuration.get("conf", "full_load_plugin_name")
cdc_plugin_name = configuration.get("conf", "cdc_plugin_name")
table_names_str = configuration.get("conf", "table_names", default="")
table_names = table_names_str.split(
",") if table_names_str.strip() != "" else []
output_location = configuration.get("conf", "output_location")
BaseDataBaseOperator.load_derived_classes()
FullLoadBase.load_derived_classes()
CDCBase.load_derived_classes()
print(database_operator_name, database_operator_type, load_type,
full_load_plugin_name, cdc_plugin_name, table_names_str, table_names,
output_location)
database_operator = BaseDataBaseOperator.get_object(
database_operator_type, database_operator_name)
database_operator_params = {
"connection_string": connection_string,
"load_type": LoadType[load_type],
"table_names": table_names,
"full_load_plugin_name": full_load_plugin_name,
"cdc_plugin_name": cdc_plugin_name,
"output_location": output_location,
}
retry_times = int(configuration.get("conf", "retry_times", 0))
run_database_operator(database_operator, database_operator_params,
retry_times)
def test_cdc_base_plugin_parameters(self):
full_load_init_params = {
"output_folder_location": self.output_folder,
"connection_string": self.postgres_connection_string,
"table_name": "public.employee",
"notify_on_completion": None
}
full_load_object = FullLoadBase(**full_load_init_params)
self.assertEqual(
list(full_load_object.__dict__.keys()).sort(), [
"connection_string", "output_folder_location", "table_name",
"notify_on_completion"
"configuration", "logger"
].sort())
def _run_full_load_process_for_table(full_load_plugin_name: str,
full_load_init_params: Dict) -> None:
"""
Worker process for Full load process to complete
:param full_load_plugin_name: plugin to be used
:type full_load_plugin_name: str
:param full_load_init_params:
:type full_load_init_params:
"""
full_load_plugin = FullLoadBase.get_object(full_load_plugin_name)
plugin_parameters = full_load_plugin.plugin_parameters
full_load_init_params = \
PostgresOperator.append_plugin_parameter_from_configuration(
full_load_init_params,
plugin_parameters
)
create_rotating_log(f"full_load/{full_load_init_params['table_name']}",
"siirto.log",
"siirto")
full_load_object = full_load_plugin(**full_load_init_params)
full_load_object.setup_graceful_shutdown()
full_load_object.execute()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.full_load_plugin = FullLoadBase.get_object(self.full_load_plugin_name)
self.cdc_plugin = CDCBase.get_object(self.cdc_plugin_name)
self._validate_input_parameters()