task0 = SSHExecuteOperator(task_id="sync_jars",
bash_command=sync_jars,
ssh_hook=sshHook,
dag=dag)
task1 = SSHExecuteOperator(task_id="import_nation",
bash_command=import_nation,
ssh_hook=sshHook,
dag=dag)
task2 = SSHExecuteOperator(task_id="import_region",
bash_command=import_region,
ssh_hook=sshHook,
dag=dag)
task3 = SSHExecuteOperator(task_id="spark_dimNation",
bash_command=spark_dimNation,
ssh_hook=sshHook,
dag=dag)
task4 = SSHExecuteOperator(task_id="export_dimNation",
bash_command=export_dimNation,
ssh_hook=sshHook,
dag=dag)
task0.set_downstream(task1)
task1.set_downstream(task2)
task2.set_downstream(task3)
task3.set_downstream(task4)
task5 = SSHExecuteOperator(task_id="export_dimCustomer",
bash_command=export_dimCustomer,
ssh_hook=sshHook,
dag=dag)
task6 = SSHExecuteOperator(task_id="export_dimSupplier",
bash_command=export_dimSupplier,
ssh_hook=sshHook,
dag=dag)
task7 = SSHExecuteOperator(task_id="export_dimPart",
bash_command=export_dimPart,
ssh_hook=sshHook,
dag=dag)
task8 = SSHExecuteOperator(task_id="export_dimDate",
bash_command=export_dimDate,
ssh_hook=sshHook,
dag=dag)
task9 = SSHExecuteOperator(task_id="export_salesFact",
bash_command=export_salesFact,
ssh_hook=sshHook,
dag=dag)
task0.set_downstream(task1)
task1.set_downstream(task2)
task3.set_upstream(task2)
task3.set_downstream(
task_or_task_list=[task4, task5, task6, task7, task8, task9])
bash_command='(bash {path}/ec_iteminfo_import.sh {last_par})'.format(path=path,last_par=iteminfo_last_update()),
ssh_hook=sshHook,
dag=dag)
spark_shop = SSHExecuteOperator(
task_id="ec_shopinfo_spark",
bash_command='(bash {path}/ec_shopinfo_parse.sh)'.format(path=path),
ssh_hook=sshHook,
dag=dag)
hive_shop= SSHExecuteOperator(
task_id="ec_shopinfo_hive",
bash_command='(bash {path}/ec_shopinfo_import.sh {last_par})'.format(path=path,last_par=shopinfo_last_update()),
ssh_hook=sshHook,
dag=dag)
final_ops= SSHExecuteOperator(
task_id="ec_hdfs_ops",
bash_command='(bash {path}/ec_item_shop_final_ops.sh)'.format(path=path),
ssh_hook=sshHook,
dag=dag)
email = EmailOperator(task_id='ec_item_and_shopinfo_email',
to=['*****@*****.**'],
subject='ec item and shop info workflow',
html_content='Execute the jobs successfully!!!!',
dag=dag)
spark_item.set_downstream(hive_item)
hive_item.set_downstream(spark_shop)
spark_shop.set_downstream(hive_shop)
hive_shop.set_downstream(final_ops)
final_ops.set_downstream(email)