# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from airflow.operators import ShortCircuitOperator, DummyOperator from airflow.models import DAG import airflow.utils.helpers from datetime import datetime, timedelta seven_days_ago = datetime.combine(datetime.today() - timedelta(7), datetime.min.time()) args = { 'owner': 'airflow', 'start_date': seven_days_ago, } dag = DAG(dag_id='example_short_circuit_operator', default_args=args) cond_true = ShortCircuitOperator(task_id='condition_is_True', python_callable=lambda: True, dag=dag) cond_false = ShortCircuitOperator(task_id='condition_is_False', python_callable=lambda: False, dag=dag) ds_true = [DummyOperator(task_id='true_' + str(i), dag=dag) for i in [1, 2]] ds_false = [DummyOperator(task_id='false_' + str(i), dag=dag) for i in [1, 2]] airflow.utils.helpers.chain(cond_true, *ds_true) airflow.utils.helpers.chain(cond_false, *ds_false)
'retries': 1, 'retry_delay': timedelta(minutes=5), } with models.DAG( dag_id='your dag name', # Continue to run DAG once per day schedule_interval=schedule_interval_dag, catchup=True, default_args=default_dag_args) as dag: email_to_gcs = GmailToGCS.ExtractAttachment(task_id='email_to_gcs', inbox_name=inbox_name_dag) checkforfile = ShortCircuitOperator(task_id='checkforfile', provide_context=False, python_callable=checkforfile) gcs_to_bq = StorageToBQ.StorageToBigQuery( task_id='gcs_to_bq', dataset_name=dataset_name_dag, bigquery_table_name=bigquery_table_name_dag, write_mode=write_mode_dag) check_dups = CheckDupBQ.CheckBQDuplication( task_id='check_dups', dataset_name=dataset_name_dag, bigquery_table_name=bigquery_table_name_dag, bigquery_table_key=bigquery_table_key_dag, date_column=date_column_dag)
'email': ['*****@*****.**'], 'email_on_failure': True, 'email_on_retry': True, 'retries': 0, } with DAG( 'PriceAlerter', default_args=default_args, schedule_interval='*/5 9-17 * * *', catchup=False, ) as dag: price_listener = PythonOperator( task_id='listener', python_callable=listener.listen, ) email_trigger = ShortCircuitOperator( task_id='email_trigger', python_callable=lambda: True if listener.trigger else False, trigger_rule=TriggerRule.NONE_FAILED, ) email = EmailOperator( task_id='email', to=email_service.email_list, subject=email_service.get_email_subject(listener.summary), html_content=email_service.get_html_content(listener.summary), ) price_listener.set_downstream(email_trigger) email_trigger.set_downstream(email)
cmd = 'ls -l' run_this_first = DummyOperator(task_id='run_this_first', dag=dag) # Define ld variables task_name_ld = "/home/etl/APP/UPCSS/LD" # Define sp variables task_name_sp = "/home/etl/APP/UPCSS/SP" # Define ul variables task_name_ul = "/home/etl/APP/UPCSS/UL" # The contents of the sp directory options = os.listdir(task_name_sp) branching = ShortCircuitOperator( task_id='branching', python_callable=lambda: random.choice(options), dag=dag) branching.set_upstream(run_this_first) join = DummyOperator(task_id='join', trigger_rule='all_success', dag=dag) for option in options: # Get ld file path file_path_ld = task_name_ld + '/' + option + '/bin/' # The contents of the obtained file_path_ld directory file_names_ld = os.listdir(file_path_ld) # Get sp file path file_path_sp = task_name_sp + '/' + option + '/bin/' # The contents of the obtained file_path_sp directory
def load_orders_data_to_csv(**kwargs): engine = PostgresHook(postgres_conn_id='tutorial_local').get_sqlalchemy_engine() data = pd.read_csv(JOINED_DATA_FILE_PATH) df = pd.DataFrame(data) df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) df.to_sql( 'orders_table', engine, if_exists='replace', index=False, ) postgres_check = ShortCircuitOperator( task_id='postgres_check', provide_context=False, python_callable=check_postgresql_available, dag=dag, ) get_customers_csv = PythonOperator( task_id='get_customers_csv', provide_context=True, python_callable=export_postgres_data_to_csv, op_kwargs={'filepath': CUSTOMERS_FILE_PATH, 'query': CUSTOMERS_QUERY}, dag=dag, ) get_goods_csv = PythonOperator( task_id='get_goods_csv', provide_context=True, python_callable=export_postgres_data_to_csv,