dag = DAG(dag_name,
catchup=False,
default_args=default_args,
schedule_interval="30 8 * * * *")
# the following tasks are created by instantiating operators
detect_file = SFTPSensor(task_id='detect_file',
poke_interval=10,
timeout=3600,
sftp_conn_id='sftp_default',
path=source_path + filename,
dag=dag)
update_nb_of_chunks = SFTPUpdateNbOfChunksOperator(
task_id='update_nb_of_chunks',
conn_id='sftp_default',
file_path=source_path + filename,
master_variable=dag_name,
chunks_variable_name="number_of_chunks",
chunk_size=chunk_size,
dag=dag)
# These are passed in as args. Seems that they are not sent : airflow bug.
dag.start_date = default_args['start_date']
trigger_single_file_concurrent_transfer = TriggerDagRunOperator(
task_id='trigger_single_file_concurrent_transfer',
trigger_dag_id=dag_to_be_triggered)
detect_file >> update_nb_of_chunks >> trigger_single_file_concurrent_transfer
dag_layer_1 = DAG(dag_id='hw_bash_layer_1',
default_args=default_args,
schedule_interval=timedelta(1))
dag_layer_2 = DAG(dag_id='hw_bash_layer_2',
default_args=default_args,
schedule_interval=None)
dag_layer_3 = DAG(dag_id='hw_bash_layer_3',
default_args=default_args,
schedule_interval=None)
# Set start data
# These are passed in as args. Seems that they aren't sent that way is a bug.
dag_layer_1.start_date = default_args['start_date']
dag_layer_2.start_date = default_args['start_date']
dag_layer_3.start_date = default_args['start_date']
# This path is used in the code below. This should identify where the code is
# being executed from.
path = '/Users/johnaven/Sandbox/bash_dag_example'
# STDOUT 'Hello World' with redirect to out.txt
create_file = BashOperator(
task_id='save-bash',
bash_command='echo "Hello John" > {path}/out_tr.txt'.format(path=path))
# print the contents of out.txt to STDOUT
print_file = BashOperator(
task_id='print-file',