"type": "INTEGER" }, { "mode": "NULLABLE", "name": "upvote_ratio", "type": "FLOAT" }, { "mode": "NULLABLE", "name": "upvote_category", "type": "INTEGER" }, { "mode": "NULLABLE", "name": "num_comments", "type": "INTEGER" }, { "mode": "NULLABLE", "name": "comments_per_upvote", "type": "NUMERIC" }, { "mode": "NULLABLE", "name": "permalink", "type": "STRING" }]) submit_reddit.dag = dag submit_reddit.set_downstream(bq_load_submissions) bq_load_submissions.set_downstream(submit_pyspark) submit_pyspark.set_downstream(bq_load_analysis)
from airflow import models, DAG from datetime import datetime, timedelta from airflow.contrib.operators.dataproc_operator import DataProcPySparkOperator from airflow.models import * BUCKET = "gs://your_bucket_name" PYSPARK_JOB = BUCKET + "/sparksql-hive/hivesql-etl.py" DEFAULT_DAG_ARGS = { "owner": "airflow", "depends_on_past": True, "start_date": datetime.utcnow(), "email_on_failure": False, "retries": 1, "retry_delay": timedelta(minutes=5), "project_id": "your_project_id", "schedule_interval": "*/15 * * * * " } with DAG("visits_by_category", default_args=DEFAULT_DAG_ARGS) as dag: submit_pyspark = DataProcPySparkOperator(task_id="run_sparksql_etl", main=PYSPARK_JOB, cluster_name="hive-sql", region="us-central1") submit_pyspark.dag = dag