task_id='UpdateWarehouse',
python_callable=update_data_warehouse,
requirements=[
'sendgrid==6.4.8', 'apache-airflow', 'psycopg2-binary',
'google-cloud-bigquery', 'google-cloud-bigquery-storage', 'pandas',
'pyarrow', 'datetime', 'pandas_gbq', 'tqdm', 'google-cloud-storage',
'fsspec', 'sklearn', 'gcsfs', 'cloudstorage'
],
python_version='3',
trigger_rule='all_done',
dag=dag)
Join = DummyOperator(task_id='Join', dag=dag, trigger_rule='all_done')
Skip1 = DummyOperator(task_id='Skip1', dag=dag, trigger_rule='all_done')
Skip2 = DummyOperator(task_id='Skip2', dag=dag, trigger_rule='all_done')
Skip3 = DummyOperator(task_id='Skip3', dag=dag, trigger_rule='all_done')
TrainModel.set_upstream(CheckTrainApi)
Skip1.set_upstream(CheckTrainApi)
CheckPredictProfile.set_upstream(Skip1)
CheckPredictProfile.set_upstream(TrainModel)
PredictProfile.set_upstream(CheckPredictProfile)
Skip2.set_upstream(CheckPredictProfile)
CheckUpdateWarehouse.set_upstream(Skip2)
CheckUpdateWarehouse.set_upstream(PredictProfile)
UpdateWarehouse.set_upstream(CheckUpdateWarehouse)
Skip3.set_upstream(CheckUpdateWarehouse)
Join.set_upstream(Skip3)
Join.set_upstream(UpdateWarehouse)
#PredictProfile.set_upstream(TrainModel)
#UpdateWarehouse.set_upstream(PredictProfile)
task_id='CalculateProbability',
python_callable=calculate_probability,
requirements=[
'sendgrid==6.4.8', 'apache-airflow', 'psycopg2-binary',
'google-cloud-bigquery', 'google-cloud-bigquery-storage', 'pandas',
'pyarrow', 'datetime', 'pandas_gbq', 'tqdm', 'google-cloud-storage',
'fsspec', 'gcsfs'
],
python_version='3',
trigger_rule='all_done',
dag=dag)
Join = DummyOperator(task_id='Join', dag=dag, trigger_rule='all_done')
Skip1 = DummyOperator(task_id='Skip1', dag=dag, trigger_rule='all_done')
Skip2 = DummyOperator(task_id='Skip2', dag=dag, trigger_rule='all_done')
Skip3 = DummyOperator(task_id='Skip3', dag=dag, trigger_rule='all_done')
CallDividendApi.set_upstream(CheckCallApi)
Skip1.set_upstream(CheckCallApi)
CheckCsvLoad.set_upstream(Skip1)
CheckCsvLoad.set_upstream(CallDividendApi)
CsvLoad.set_upstream(CheckCsvLoad)
Skip2.set_upstream(CheckCsvLoad)
CheckCalculateProbability.set_upstream(Skip2)
CheckCalculateProbability.set_upstream(CsvLoad)
CalculateProbability.set_upstream(CheckCalculateProbability)
Skip3.set_upstream(CheckCalculateProbability)
Join.set_upstream(Skip3)
Join.set_upstream(CalculateProbability)
#CsvLoad.set_upstream(CallDividendApi)
#CalculateProbability.set_upstream(CsvLoad)