def etl(ctx: Context, schema: str, renew: bool, table: List[str]): alias = ctx.obj["alias"] tables = table if not tables: tables = Settings.get_source_db_database_tables_name(alias, schema) tables_pk = {} reader = get_reader(alias) for table in tables: tables_pk[table] = reader.get_primary_key(schema, table) etl_full(alias, schema, tables_pk, renew)
def check(ctx: Context, schema: str): alias = ctx.obj["alias"] reader = get_reader(alias) writer = get_writer() tables = Settings.get_source_db_database_tables_name(alias, schema) for table in tables: source_table_count = reader.get_count(schema, table) target_table_count = writer.get_count(schema, table) if source_table_count == target_table_count: logger.info(f"{schema}.{table} is equal, count={source_table_count}") else: logger.warning( f"{schema}.{table} is not equal, source_table_count={source_table_count}, target_table_count={target_table_count}" )
def consume(ctx: Context, schema: str, skip_error: bool, last_msg_id: str): alias = ctx.obj["alias"] reader = get_reader(alias) tables = Settings.get_source_db_database_tables_name(alias, schema) tables_pk = {} for table in tables: tables_pk[table] = reader.get_primary_key(schema, table) # try etl full etl_full(alias, schema, tables_pk) table_dict = Settings.get_source_db_database_tables_dict(alias, schema) continuous_etl( alias, schema, tables_pk, table_dict, last_msg_id, skip_error, )