dag = DAG(dag_name,
          catchup=False,
          default_args=default_args,
          schedule_interval="30 8 * * * *")

# the following tasks are created by instantiating operators
detect_file = SFTPSensor(task_id='detect_file',
                         poke_interval=10,
                         timeout=3600,
                         sftp_conn_id='sftp_default',
                         path=source_path + filename,
                         dag=dag)

update_nb_of_chunks = SFTPUpdateNbOfChunksOperator(
    task_id='update_nb_of_chunks',
    conn_id='sftp_default',
    file_path=source_path + filename,
    master_variable=dag_name,
    chunks_variable_name="number_of_chunks",
    chunk_size=chunk_size,
    dag=dag)

# These are passed in as args. Seems that they are not sent : airflow bug.
dag.start_date = default_args['start_date']

trigger_single_file_concurrent_transfer = TriggerDagRunOperator(
    task_id='trigger_single_file_concurrent_transfer',
    trigger_dag_id=dag_to_be_triggered)

detect_file >> update_nb_of_chunks >> trigger_single_file_concurrent_transfer
Exemplo n.º 2
0
dag_layer_1 = DAG(dag_id='hw_bash_layer_1',
                  default_args=default_args,
                  schedule_interval=timedelta(1))

dag_layer_2 = DAG(dag_id='hw_bash_layer_2',
                  default_args=default_args,
                  schedule_interval=None)

dag_layer_3 = DAG(dag_id='hw_bash_layer_3',
                  default_args=default_args,
                  schedule_interval=None)

# Set start data

# These are passed in as args. Seems that they aren't sent that way is a bug.
dag_layer_1.start_date = default_args['start_date']
dag_layer_2.start_date = default_args['start_date']
dag_layer_3.start_date = default_args['start_date']

# This path is used in the code below. This should identify where the code is
# being executed from.
path = '/Users/johnaven/Sandbox/bash_dag_example'

# STDOUT 'Hello World' with redirect to out.txt
create_file = BashOperator(
    task_id='save-bash',
    bash_command='echo "Hello John" > {path}/out_tr.txt'.format(path=path))

# print the contents of out.txt to STDOUT
print_file = BashOperator(
    task_id='print-file',