hrrr = HRRR(docker_compose_file, config_file) grib2nc = hrrr.get_compose_grib2nc(yesterday=True) # upload to swift, use the execution date swift = HRRR('/home/ubuntu/config/docker-compose-swift.yml', config_file) upload_swift = swift.get_compose_upload_swift(log_file='/data/hrrr/logs/swift_historical.log', yesterday=True) # delete the directory delete_dir = hrrr.delete_hrrr_directory() t1 = SSHOperator( ssh_conn_id='ssh_hrrr', task_id='convert_grib2nc', command=grib2nc, dag=dag) t2 = SSHOperator( ssh_conn_id='ssh_hrrr', task_id='upload_swift', command=upload_swift, dag=dag) t3 = SSHOperator( ssh_conn_id='ssh_hrrr', task_id='delete_empty_directory', command=delete_dir, dag=dag) t1.set_downstream(t2) t2.set_downstream(t3)
files=[file1.name, file2.name]) email_op.execute(context) file1.close() file2.close() notification = PythonOperator(task_id="notification", python_callable=build_email, provide_context=True, dag=dag) task1 = SSHOperator( task_id='switch2ALhome', bash_command= "cd /home/pchoix/BM3 && ./BM3.py runjob -p client1 -j ingestion", dag=dag) ssh_task = SSHOperator( ssh_conn_id='ssh_default', task_id='ssh_to_other_host', command='cd /home/pchoix/BM3;./BM3.py runjob -p client1 -j ingestion', dag=dag) task1.set_downstream(notification) if __name__ == "__main__": dag.cli() #airflow test test_bm3_ssh ssh_to_other_host 2001-01-01 works! ssh_default must be pre-created in Airflow/Admin/Connections
dag=dag) task_06_marketing_data_reporter = SSHOperator( task_id='06_marketing_data_reporter', ssh_conn_id='ssh_default', command='/bin/bash /tmp/test.sh 06', timeout=10, dag=dag) task_07_bend_reporter = SSHOperator(task_id='07_bend_reporter', ssh_conn_id='ssh_default', command='/bin/bash /tmp/test.sh 07', timeout=10, dag=dag) online_task01_onions_data_soruce.set_downstream( online_task02_orders_data_soruce) online_task02_orders_data_soruce.set_downstream( task_01_prepare_underlying_data_source) task_01_prepare_underlying_data_source.set_downstream( task_02_users_collection_monitor_data_source) task_02_users_collection_monitor_data_source.set_downstream( task_03_tianji_underlying_data_source) task_02_users_collection_monitor_data_source.set_downstream( task_04_etl_underlying_data_source) task_03_tianji_underlying_data_source.set_downstream( task_05_videology_data_source) task_03_tianji_underlying_data_source.set_downstream( task_06_marketing_data_reporter) task_03_tianji_underlying_data_source.set_downstream(task_07_bend_reporter) task_07_bend_reporter.set_downstream(sleep_3_sec)