# Global variables bqproject = 'usage-data-reporting' datasetenv = 'DEV' # Create DAG dag = DAG(ENVIRONMENTS['dev']['dag-name'], default_args=DEFAULT_ARGS, schedule_interval=schedule_interval, description='CK DAG GBQ Test') MOVE_LDZ_DATA_TO_DWH = BigQueryOperator( dag=dag, task_id='CK_GBQ_TEST_TASK_01', sql='Tests/ckgbqtest.sql', params={ "project": bqproject, "environment": datasetenv }, # destination_dataset_table=bqproject + '.' + datasetenv + '_DWH_GBQ_Test.CK_GBQ_Test', # Zieltabelle write_disposition= 'WRITE_APPEND', # Specify the write disposition truncate or write_append use_legacy_sql=False, bigquery_conn_id=ENVIRONMENTS['dev']['connection-id']) # Referenziert auf den dag - name (s.o.) - liefert eine Beschreibung mit. MOVE_LDZ_DATA_TO_DWH.doc_md = """Write data from LDZ to DWH""" # Define how the different steps in the workflow are executed MOVE_LDZ_DATA_TO_DWH
task_id='my_bq_task_1_' + lob, # task id's must be uniqe within the dag bql= 'my_qry_1.sql', # the actual sql command we want to run on bigquery is in this file in the same folder. it is also templated params={ "lob": lob }, # the sql file above have a template in it for a 'lob' paramater - this is how we pass it in destination_dataset_table='airflow.' + lob + '_test_task1', # we also in this example want our target table to be lob and task specific write_disposition= 'WRITE_TRUNCATE', # drop and recreate this table each time, you could use other options here bigquery_conn_id= 'my_gcp_connection' # this is the airflow connection to gcp we defined in the front end. More info here: https://github.com/alexvanboxel/airflow-gcp-examples ) # add documentation for what this task does - this will be displayed in the Airflow UI bq_task_1.doc_md = """\ Append a "Hello World!" message string to the table [airflow.<lob>_test_task1] """ # define the second task, in our case another big query operator bq_task_2 = BigQueryOperator( dag= dag, # need to tell airflow that this task belongs to the dag we defined above task_id='my_bq_task_2_' + lob, # task id's must be uniqe within the dag bql= 'my_qry_2.sql', # the actual sql command we want to run on bigquery is in this file in the same folder. it is also templated params={ "lob": lob }, # the sql file above have a template in it for a 'lob' paramater - this is how we pass it in destination_dataset_table='airflow.' + lob +