Python Settings.get_source_db_database_tables_nameの例

コード例 #1

ファイル: cli.py プロジェクト: yy1117/synch

def etl(ctx: Context, schema: str, renew: bool, table: List[str]):
    alias = ctx.obj["alias"]
    tables = table
    if not tables:
        tables = Settings.get_source_db_database_tables_name(alias, schema)
    tables_pk = {}
    reader = get_reader(alias)
    for table in tables:
        tables_pk[table] = reader.get_primary_key(schema, table)
    etl_full(alias, schema, tables_pk, renew)

コード例 #2

def check(ctx: Context, schema: str):
    alias = ctx.obj["alias"]
    reader = get_reader(alias)
    writer = get_writer()
    tables = Settings.get_source_db_database_tables_name(alias, schema)
    for table in tables:
        source_table_count = reader.get_count(schema, table)
        target_table_count = writer.get_count(schema, table)
        if source_table_count == target_table_count:
            logger.info(f"{schema}.{table} is equal, count={source_table_count}")
        else:
            logger.warning(
                f"{schema}.{table} is not equal, source_table_count={source_table_count}, target_table_count={target_table_count}"
            )

コード例 #3

ファイル: cli.py プロジェクト: zhbdesign/synch

def consume(ctx: Context, schema: str, skip_error: bool, last_msg_id: str):
    alias = ctx.obj["alias"]
    reader = get_reader(alias)
    tables = Settings.get_source_db_database_tables_name(alias, schema)
    tables_pk = {}
    for table in tables:
        tables_pk[table] = reader.get_primary_key(schema, table)

    # try etl full
    etl_full(alias, schema, tables_pk)
    table_dict = Settings.get_source_db_database_tables_dict(alias, schema)

    continuous_etl(
        alias, schema, tables_pk, table_dict, last_msg_id, skip_error,
    )