"retries": 1,
"retry_delay": timedelta(minutes=10)
}
schedule_interval = timedelta(days=1)
date = datetime.today().strftime("%Y-%m-%d")
dag = DAG(f"TT-TEST_series_series_{date}",
default_args=original_args,
schedule_interval=schedule_interval)
### 並列処理タスクテスト START
interval = create_interval("daily", 1)
destination_dataset_table = "gree-ua-kpi-dev:sample_takeuchi_dataset_tokyo.q_dau"
sql1 = get_sql("TT_TEST/qp03_01.sql")
query_params_01 = [{
"name": "target_date",
"parameterType": {
"type": "DATE"
},
"parameterValue": {
"value": "2020-10-01"
}
}]
sql2 = get_sql("TT_TEST/qp03_02.sql")
query_params_02 = [{
"name": "dt",
"parameterType": {
"type": "DATE"
from airflow.models import Variable
sys.path.append(Variable.get('module_path'))
from functions import get_dag, get_bq_operator, get_sql
from airflow import DAG
today = datetime.today()
yesterday = today - timedelta(days=1)
dag = get_dag("DAG-002")
destination_dataset_table = "gree-ua-kpi-dev:sample_takeuchi_dataset_tokyo.q_dau"
params = {"from_date": "2020-10-01", "to_date": "2020-12-30"}
sql = get_sql("test/test_a2b.sql", params)
op_1 = get_bq_operator(dag,
task_id="task1",
sql=sql,
gcp_conn_id="gree-anger-dev-bigquery")
op_2 = get_bq_operator(dag,
task_id="task2",
sql=sql,
gcp_conn_id="gree-anger-dev-bigquery")
op_3 = get_bq_operator(dag,
task_id="task3",
sql=sql,
gcp_conn_id="gree-anger-dev-bigquery",
import traceback
# from airflow.contrib.operators.bigquery_check_operator import BigQueryCheckOperator
# sys.path.append(os.path.join('/home/airflow/gcs/plugins/modules/'))
from airflow.models import Variable
sys.path.append(Variable.get('module_path'))
# sys.path.append(os.getenv('MODULE_PATH'))
from functions import get_sql, get_dag, get_bq_operator, slack_failure_notification, slack_success_notification
today = datetime.today()
yesterday = today - timedelta(days=1)
target_date = yesterday.strftime('%Y-%m-%d')
params = {"table": "`gree-anger-dev.dummy_anger.access`", "target_date": "2020-12-07", "page_group": "home"}
sql = get_sql("test/test.sql", params)
# bq_service = BqOpServiceClass("DAG-test")
dag = get_dag("DAG-TEST-001")
task_1 = get_bq_operator(
dag,
task_id = "test_task_1",
sql = sql,
gcp_conn_id = "gree-anger-dev-bigquery",
on_failure_callback=slack_failure_notification,
on_success_callback=slack_success_notification)
task_2 = get_bq_operator(
dag,
one_day_ago = datetime.combine(datetime.today() - timedelta(1), datetime.min.time())
original_args = {
"owner": "airflow",
"depends_on_past": False,
"start_date": one_day_ago,
"email": ["*****@*****.**"],
"email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知
"email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知
"retries": 1,
"retry_delay": timedelta(minutes=10)}
schedule_interval = timedelta(days=1)
dag = DAG("TT-TEST_QP01_2021-04-05", default_args=original_args, schedule_interval=schedule_interval)
sql = get_sql("TT_TEST/get_data_query_params.sql")
query_params = [
{
"name": "from_date",
"parameterType": {
"type": "DATE"
},
"parameterValue": {
"value": "2020-10-01"
}
},
{
"name": "end_date",
"parameterType": {
"type": "DATE"
},
one_day_ago = datetime.combine(datetime.today() - timedelta(1), datetime.min.time())
original_args = {
"owner": "airflow",
"depends_on_past": False,
"start_date": one_day_ago,
"email": ["*****@*****.**"],
"email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知
"email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知
"retries": 1,
"retry_delay": timedelta(minutes=10)}
schedule_interval = timedelta(days=1)
dag = DAG("TT-TEST_GET01_2021-04-05", default_args=original_args, schedule_interval=schedule_interval)
sql = get_sql("TT_TEST/get_data.sql")
dumy01 = DummyOperator(task_id='dummy01', dag = dag)
dumy02 = DummyOperator(task_id='dummy02', dag = dag)
task01 = get_bq_operator_as_get_data(
task_id = "get01",
dag = dag,
sql = sql,
gcp_conn_id = "gree-anger-dev-bigquery")
dumy01 >> task01 >> dumy02
"depends_on_past": False,
"start_date": one_day_ago,
"email": ["*****@*****.**"],
"email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知
"email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知
"retries": 1,
"retry_delay": timedelta(minutes=10)}
schedule_interval = timedelta(days=1)
dag = DAG("TEST_PRJ-T003", default_args=original_args, schedule_interval=schedule_interval)
interval = create_interval("monthly", 1)
destination_dataset_table = "destination_dataset_table: your_project_id.dataset.table${{ params.partition_date }}"
sql1 = get_sql("TEST_PRJ/test2.sql")
query_params_01 = [
{
"name": "dt",
"parameterType": {
"type": "DATE"
},
"parameterValue": {
"value": "{{ params.target_date }}"
}
}
]
task01 = get_big_query_task_model(
task_id = "task01",
dag = dag,
"depends_on_past": False,
"start_date": one_day_ago,
"email": ["*****@*****.**"],
"email_on_failure": False, # Trueにするとtaskが失敗した際にメールが通知
"email_on_retry": False, # Trueにするとtaskがリトライが発生した際にメールが通知
"retries": 1,
"retry_delay": timedelta(minutes=10)}
schedule_interval = timedelta(days=1)
dag = DAG("TT-TEST-001", default_args=original_args, schedule_interval=schedule_interval)
interval = create_interval("daily", 1)
destination_dataset_table = "gree-ua-kpi-dev:sample_takeuchi_dataset_tokyo.q_dau"
sql1 = get_sql("TT_TEST/qp03_01.sql")
query_params_01 = [
{
"name": "target_date",
"parameterType": {
"type": "DATE"
},
"parameterValue": {
"value": "2020-10-01"
}
}
]
task01 = get_big_query_task_model(
task_id = "qp03_01",
dag = dag,