Ejemplos de get_s3_to_redshift_dag en Python

Ejemplo n.º 1

start_date = datetime.datetime.utcnow()

dag = DAG(
    "lesson3.exercise3",
    start_date=start_date,
)

trips_task_id = "trips_subdag"
trips_subdag_task = SubDagOperator(
    subdag=get_s3_to_redshift_dag(
        "lesson3.exercise3",
        trips_task_id,
        "redshift",
        "aws_credentials",
        "trips",
        sql_statements.CREATE_TRIPS_TABLE_SQL,
        s3_bucket="udac-data-pipelines",
        s3_key="divvy/unpartitioned/divvy_trips_2018.csv",
        start_date=start_date,
    ),
    task_id=trips_task_id,
    dag=dag,
)

stations_task_id = "stations_subdag"
stations_subdag_task = SubDagOperator(
    subdag=get_s3_to_redshift_dag(
        "lesson3.exercise3",
        stations_task_id,
        "redshift",

Ejemplo n.º 2

Archivo: exercise4.py Proyecto: coetzeevs/udacity

start_date = datetime.datetime.utcnow()

dag = DAG(
    "lesson3.exercise4",
    start_date=start_date,
)

copy_trips_task_id = "copy_trips_subdag"
copy_trips_subdag_task = SubDagOperator(
    subdag=get_s3_to_redshift_dag(
        parent_dag_name="lesson3.exercise4",
        task_id=copy_trips_task_id,
        redshift_conn_id="redshift",
        aws_credentials_id="aws_credentials",
        table="trips",
        create_sql_stmt=sql_statements.CREATE_TRIPS_TABLE_SQL,
        s3_bucket="udacity-dend",
        s3_prefix="data-pipelines",
        s3_key="divvy/unpartitioned/divvy_trips_2018.csv",
        start_date=start_date,
    ),
    task_id=copy_trips_task_id,
    dag=dag,
)

calculate_facts_task = FactsCalculatorOperator(
    dag=dag,
    task_id="calculate_facts",
    redshift_conn_id="redshift",
    origin_table="trips",
    destination_table="fact_tripduration",