hrrr = HRRR(docker_compose_file, config_file)
grib2nc = hrrr.get_compose_grib2nc(yesterday=True)
# upload to swift, use the execution date
swift = HRRR('/home/ubuntu/config/docker-compose-swift.yml', config_file)
upload_swift = swift.get_compose_upload_swift(log_file='/data/hrrr/logs/swift_historical.log', yesterday=True)
# delete the directory
delete_dir = hrrr.delete_hrrr_directory()
t1 = SSHOperator(
ssh_conn_id='ssh_hrrr',
task_id='convert_grib2nc',
command=grib2nc,
dag=dag)
t2 = SSHOperator(
ssh_conn_id='ssh_hrrr',
task_id='upload_swift',
command=upload_swift,
dag=dag)
t3 = SSHOperator(
ssh_conn_id='ssh_hrrr',
task_id='delete_empty_directory',
command=delete_dir,
dag=dag)
t1.set_downstream(t2)
t2.set_downstream(t3)
files=[file1.name, file2.name])
email_op.execute(context)
file1.close()
file2.close()
notification = PythonOperator(task_id="notification",
python_callable=build_email,
provide_context=True,
dag=dag)
task1 = SSHOperator(
task_id='switch2ALhome',
bash_command=
"cd /home/pchoix/BM3 && ./BM3.py runjob -p client1 -j ingestion",
dag=dag)
ssh_task = SSHOperator(
ssh_conn_id='ssh_default',
task_id='ssh_to_other_host',
command='cd /home/pchoix/BM3;./BM3.py runjob -p client1 -j ingestion',
dag=dag)
task1.set_downstream(notification)
if __name__ == "__main__":
dag.cli()
#airflow test test_bm3_ssh ssh_to_other_host 2001-01-01 works! ssh_default must be pre-created in Airflow/Admin/Connections
dag=dag)
task_06_marketing_data_reporter = SSHOperator(
task_id='06_marketing_data_reporter',
ssh_conn_id='ssh_default',
command='/bin/bash /tmp/test.sh 06',
timeout=10,
dag=dag)
task_07_bend_reporter = SSHOperator(task_id='07_bend_reporter',
ssh_conn_id='ssh_default',
command='/bin/bash /tmp/test.sh 07',
timeout=10,
dag=dag)
online_task01_onions_data_soruce.set_downstream(
online_task02_orders_data_soruce)
online_task02_orders_data_soruce.set_downstream(
task_01_prepare_underlying_data_source)
task_01_prepare_underlying_data_source.set_downstream(
task_02_users_collection_monitor_data_source)
task_02_users_collection_monitor_data_source.set_downstream(
task_03_tianji_underlying_data_source)
task_02_users_collection_monitor_data_source.set_downstream(
task_04_etl_underlying_data_source)
task_03_tianji_underlying_data_source.set_downstream(
task_05_videology_data_source)
task_03_tianji_underlying_data_source.set_downstream(
task_06_marketing_data_reporter)
task_03_tianji_underlying_data_source.set_downstream(task_07_bend_reporter)
task_07_bend_reporter.set_downstream(sleep_3_sec)
