"retries": 1, "retry_delay": timedelta(minutes=10) } schedule_interval = timedelta(days=1) date = datetime.today().strftime("%Y-%m-%d") dag = DAG(f"TT-TEST_series_series_{date}", default_args=original_args, schedule_interval=schedule_interval) ### 並列処理タスクテスト START interval = create_interval("daily", 1) destination_dataset_table = "gree-ua-kpi-dev:sample_takeuchi_dataset_tokyo.q_dau" sql1 = get_sql("TT_TEST/qp03_01.sql") query_params_01 = [{ "name": "target_date", "parameterType": { "type": "DATE" }, "parameterValue": { "value": "2020-10-01" } }] sql2 = get_sql("TT_TEST/qp03_02.sql") query_params_02 = [{ "name": "dt", "parameterType": { "type": "DATE"
from airflow.models import Variable sys.path.append(Variable.get('module_path')) from functions import get_dag, get_bq_operator, get_sql from airflow import DAG today = datetime.today() yesterday = today - timedelta(days=1) dag = get_dag("DAG-002") destination_dataset_table = "gree-ua-kpi-dev:sample_takeuchi_dataset_tokyo.q_dau" params = {"from_date": "2020-10-01", "to_date": "2020-12-30"} sql = get_sql("test/test_a2b.sql", params) op_1 = get_bq_operator(dag, task_id="task1", sql=sql, gcp_conn_id="gree-anger-dev-bigquery") op_2 = get_bq_operator(dag, task_id="task2", sql=sql, gcp_conn_id="gree-anger-dev-bigquery") op_3 = get_bq_operator(dag, task_id="task3", sql=sql, gcp_conn_id="gree-anger-dev-bigquery",
import traceback # from airflow.contrib.operators.bigquery_check_operator import BigQueryCheckOperator # sys.path.append(os.path.join('/home/airflow/gcs/plugins/modules/')) from airflow.models import Variable sys.path.append(Variable.get('module_path')) # sys.path.append(os.getenv('MODULE_PATH')) from functions import get_sql, get_dag, get_bq_operator, slack_failure_notification, slack_success_notification today = datetime.today() yesterday = today - timedelta(days=1) target_date = yesterday.strftime('%Y-%m-%d') params = {"table": "`gree-anger-dev.dummy_anger.access`", "target_date": "2020-12-07", "page_group": "home"} sql = get_sql("test/test.sql", params) # bq_service = BqOpServiceClass("DAG-test") dag = get_dag("DAG-TEST-001") task_1 = get_bq_operator( dag, task_id = "test_task_1", sql = sql, gcp_conn_id = "gree-anger-dev-bigquery", on_failure_callback=slack_failure_notification, on_success_callback=slack_success_notification) task_2 = get_bq_operator( dag,
one_day_ago = datetime.combine(datetime.today() - timedelta(1), datetime.min.time()) original_args = { "owner": "airflow", "depends_on_past": False, "start_date": one_day_ago, "email": ["*****@*****.**"], "email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知 "email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知 "retries": 1, "retry_delay": timedelta(minutes=10)} schedule_interval = timedelta(days=1) dag = DAG("TT-TEST_QP01_2021-04-05", default_args=original_args, schedule_interval=schedule_interval) sql = get_sql("TT_TEST/get_data_query_params.sql") query_params = [ { "name": "from_date", "parameterType": { "type": "DATE" }, "parameterValue": { "value": "2020-10-01" } }, { "name": "end_date", "parameterType": { "type": "DATE" },
one_day_ago = datetime.combine(datetime.today() - timedelta(1), datetime.min.time()) original_args = { "owner": "airflow", "depends_on_past": False, "start_date": one_day_ago, "email": ["*****@*****.**"], "email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知 "email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知 "retries": 1, "retry_delay": timedelta(minutes=10)} schedule_interval = timedelta(days=1) dag = DAG("TT-TEST_GET01_2021-04-05", default_args=original_args, schedule_interval=schedule_interval) sql = get_sql("TT_TEST/get_data.sql") dumy01 = DummyOperator(task_id='dummy01', dag = dag) dumy02 = DummyOperator(task_id='dummy02', dag = dag) task01 = get_bq_operator_as_get_data( task_id = "get01", dag = dag, sql = sql, gcp_conn_id = "gree-anger-dev-bigquery") dumy01 >> task01 >> dumy02
"depends_on_past": False, "start_date": one_day_ago, "email": ["*****@*****.**"], "email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知 "email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知 "retries": 1, "retry_delay": timedelta(minutes=10)} schedule_interval = timedelta(days=1) dag = DAG("TEST_PRJ-T003", default_args=original_args, schedule_interval=schedule_interval) interval = create_interval("monthly", 1) destination_dataset_table = "destination_dataset_table: your_project_id.dataset.table${{ params.partition_date }}" sql1 = get_sql("TEST_PRJ/test2.sql") query_params_01 = [ { "name": "dt", "parameterType": { "type": "DATE" }, "parameterValue": { "value": "{{ params.target_date }}" } } ] task01 = get_big_query_task_model( task_id = "task01", dag = dag,
"depends_on_past": False, "start_date": one_day_ago, "email": ["*****@*****.**"], "email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知 "email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知 "retries": 1, "retry_delay": timedelta(minutes=10)} schedule_interval = timedelta(days=1) dag = DAG("TT-TEST-001", default_args=original_args, schedule_interval=schedule_interval) interval = create_interval("daily", 1) destination_dataset_table = "gree-ua-kpi-dev:sample_takeuchi_dataset_tokyo.q_dau" sql1 = get_sql("TT_TEST/qp03_01.sql") query_params_01 = [ { "name": "target_date", "parameterType": { "type": "DATE" }, "parameterValue": { "value": "2020-10-01" } } ] task01 = get_big_query_task_model( task_id = "qp03_01", dag = dag,