def test_cdc_base_plugin_execute(self): cdc_init_params = { "output_folder_location": self.output_folder, "connection_string": self.postgres_connection_string, "table_names": [] } with self.assertRaises(NotImplementedError) as cm: cdc_base_object = CDCBase(**cdc_init_params) cdc_base_object.execute()
def setUpClass(cls) -> None: logging.disable(logging.CRITICAL) cls.postgres_testing = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) # setup postgres db cls.setup_postgres() # load all derived classes for the test setup CDCBase.load_derived_classes() FullLoadBase.load_derived_classes() BaseDataBaseOperator.load_derived_classes()
def test_cdc_base_plugin_empty_table(self): cdc_init_params = { "output_folder_location": self.output_folder, "connection_string": self.postgres_connection_string, "table_names": None } with self.assertRaises(ValueError) as cm: CDCBase(**cdc_init_params) self.assertEqual(str(cm.exception), "Table name is None or Empty") cdc_init_params = { "output_folder_location": self.output_folder, "connection_string": self.postgres_connection_string, "table_names": "public.employee" } with self.assertRaises(ValueError) as cm: CDCBase(**cdc_init_params) self.assertEqual(str(cm.exception), "Table name is None or Empty")
def test_cdc_base_plugin_empty_output(self): cdc_init_params = { "output_folder_location": None, "connection_string": self.postgres_connection_string, "table_names": [] } with self.assertRaises(ValueError) as cm: CDCBase(**cdc_init_params) self.assertEqual(str(cm.exception), "output_folder_location is empty")
def initialize(): """ Initialize the lineage settings """ # register interrupt signal register_ctrl_c_signal() # create the logger create_rotating_log() database_operator_name = configuration.get("conf", "database_operator") database_operator_type = configuration.get("conf", "database_operator_type") connection_string = configuration.get("conf", "connection_string") load_type = configuration.get("conf", "load_type") full_load_plugin_name = configuration.get("conf", "full_load_plugin_name") cdc_plugin_name = configuration.get("conf", "cdc_plugin_name") table_names_str = configuration.get("conf", "table_names", default="") table_names = table_names_str.split( ",") if table_names_str.strip() != "" else [] output_location = configuration.get("conf", "output_location") BaseDataBaseOperator.load_derived_classes() FullLoadBase.load_derived_classes() CDCBase.load_derived_classes() print(database_operator_name, database_operator_type, load_type, full_load_plugin_name, cdc_plugin_name, table_names_str, table_names, output_location) database_operator = BaseDataBaseOperator.get_object( database_operator_type, database_operator_name) database_operator_params = { "connection_string": connection_string, "load_type": LoadType[load_type], "table_names": table_names, "full_load_plugin_name": full_load_plugin_name, "cdc_plugin_name": cdc_plugin_name, "output_location": output_location, } retry_times = int(configuration.get("conf", "retry_times", 0)) run_database_operator(database_operator, database_operator_params, retry_times)
def test_cdc_base_plugin_parameters(self): cdc_init_params = { "output_folder_location": self.output_folder, "connection_string": self.postgres_connection_string, "table_names": ["public.employee"] } cdc_object = CDCBase(**cdc_init_params) self.assertEqual( list(cdc_object.__dict__.keys()).sort(), [ "connection_string", "output_folder_location", "table_names", "configuration", "logger" ].sort())
def _run_cdc_process(cdc_plugin_name: str, cdc_init_params: Dict) -> None: """ Worker process for cdc process to complete :param cdc_plugin_name: plugin to be used :type cdc_plugin_name: str :param cdc_init_params: :type cdc_init_params: """ cdc_plugin = CDCBase.get_object(cdc_plugin_name) plugin_parameters = cdc_plugin.plugin_parameters cdc_init_params = \ PostgresOperator.append_plugin_parameter_from_configuration( cdc_init_params, plugin_parameters ) create_rotating_log("cdc", "siirto.log", "siirto") cdc_object = cdc_plugin(**cdc_init_params) cdc_object.setup_graceful_shutdown() cdc_object.execute()
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.full_load_plugin = FullLoadBase.get_object(self.full_load_plugin_name) self.cdc_plugin = CDCBase.get_object(self.cdc_plugin_name) self._validate_input_parameters()
def test_cdc_base_plugin_load_cdc_default_plugin(self): CDCBase.load_derived_classes() child_operator_object = CDCBase.get_object("PgDefaultCDCPlugin") self.assertEqual(child_operator_object.__dict__["__module__"], "siirto.plugins.cdc.pg_default_cdc_plugin") self.assertEqual(child_operator_object.__name__, "PgDefaultCDCPlugin")