dag = DAG(dag_name, catchup=False, default_args=default_args, schedule_interval="30 8 * * * *") # the following tasks are created by instantiating operators detect_file = SFTPSensor(task_id='detect_file', poke_interval=10, timeout=3600, sftp_conn_id='sftp_default', path=source_path + filename, dag=dag) update_nb_of_chunks = SFTPUpdateNbOfChunksOperator( task_id='update_nb_of_chunks', conn_id='sftp_default', file_path=source_path + filename, master_variable=dag_name, chunks_variable_name="number_of_chunks", chunk_size=chunk_size, dag=dag) # These are passed in as args. Seems that they are not sent : airflow bug. dag.start_date = default_args['start_date'] trigger_single_file_concurrent_transfer = TriggerDagRunOperator( task_id='trigger_single_file_concurrent_transfer', trigger_dag_id=dag_to_be_triggered) detect_file >> update_nb_of_chunks >> trigger_single_file_concurrent_transfer
dag_layer_1 = DAG(dag_id='hw_bash_layer_1', default_args=default_args, schedule_interval=timedelta(1)) dag_layer_2 = DAG(dag_id='hw_bash_layer_2', default_args=default_args, schedule_interval=None) dag_layer_3 = DAG(dag_id='hw_bash_layer_3', default_args=default_args, schedule_interval=None) # Set start data # These are passed in as args. Seems that they aren't sent that way is a bug. dag_layer_1.start_date = default_args['start_date'] dag_layer_2.start_date = default_args['start_date'] dag_layer_3.start_date = default_args['start_date'] # This path is used in the code below. This should identify where the code is # being executed from. path = '/Users/johnaven/Sandbox/bash_dag_example' # STDOUT 'Hello World' with redirect to out.txt create_file = BashOperator( task_id='save-bash', bash_command='echo "Hello John" > {path}/out_tr.txt'.format(path=path)) # print the contents of out.txt to STDOUT print_file = BashOperator( task_id='print-file',