Python load_s3_data_to_intermediate_table Exemples

Exemple #1

Fichier : test_sql.py Projet : ChariniNana/cccatalog

def _load_s3_tsv(tmpdir, bucket, tsv_file_name):
    tsv_file_path = os.path.join(RESOURCES, tsv_file_name)
    key = 'path/to/object/{tsv_file_name}'
    bucket.upload_file(tsv_file_path, key)
    sql.load_s3_data_to_intermediate_table(
        POSTGRES_CONN_ID,
        S3_TEST_BUCKET,
        key,
        TEST_ID
    )

Exemple #2

Fichier : loader.py Projet : jhutchings1/cccatalog

def load_s3_data(
        bucket,
        aws_conn_id,
        postgres_conn_id,
        identifier
):
    tsv_key = s3.get_staged_s3_object(identifier, bucket, aws_conn_id)
    sql.load_s3_data_to_intermediate_table(
        postgres_conn_id,
        bucket,
        tsv_key,
        identifier
    )
    sql.upsert_records_to_image_table(postgres_conn_id, identifier)

Exemple #3

def load_s3_data(
        bucket,
        aws_conn_id,
        postgres_conn_id,
        identifier,
        overwrite=False,
):
    tsv_key = s3.get_staged_s3_object(identifier, bucket, aws_conn_id)
    sql.load_s3_data_to_intermediate_table(
        postgres_conn_id,
        bucket,
        tsv_key,
        identifier
    )
    if overwrite is True:
        sql.overwrite_records_in_image_table(postgres_conn_id, identifier)
    else:
        sql.upsert_records_to_image_table(postgres_conn_id, identifier)