task_id='UpdateWarehouse', python_callable=update_data_warehouse, requirements=[ 'sendgrid==6.4.8', 'apache-airflow', 'psycopg2-binary', 'google-cloud-bigquery', 'google-cloud-bigquery-storage', 'pandas', 'pyarrow', 'datetime', 'pandas_gbq', 'tqdm', 'google-cloud-storage', 'fsspec', 'sklearn', 'gcsfs', 'cloudstorage' ], python_version='3', trigger_rule='all_done', dag=dag) Join = DummyOperator(task_id='Join', dag=dag, trigger_rule='all_done') Skip1 = DummyOperator(task_id='Skip1', dag=dag, trigger_rule='all_done') Skip2 = DummyOperator(task_id='Skip2', dag=dag, trigger_rule='all_done') Skip3 = DummyOperator(task_id='Skip3', dag=dag, trigger_rule='all_done') TrainModel.set_upstream(CheckTrainApi) Skip1.set_upstream(CheckTrainApi) CheckPredictProfile.set_upstream(Skip1) CheckPredictProfile.set_upstream(TrainModel) PredictProfile.set_upstream(CheckPredictProfile) Skip2.set_upstream(CheckPredictProfile) CheckUpdateWarehouse.set_upstream(Skip2) CheckUpdateWarehouse.set_upstream(PredictProfile) UpdateWarehouse.set_upstream(CheckUpdateWarehouse) Skip3.set_upstream(CheckUpdateWarehouse) Join.set_upstream(Skip3) Join.set_upstream(UpdateWarehouse) #PredictProfile.set_upstream(TrainModel) #UpdateWarehouse.set_upstream(PredictProfile)
task_id='CalculateProbability', python_callable=calculate_probability, requirements=[ 'sendgrid==6.4.8', 'apache-airflow', 'psycopg2-binary', 'google-cloud-bigquery', 'google-cloud-bigquery-storage', 'pandas', 'pyarrow', 'datetime', 'pandas_gbq', 'tqdm', 'google-cloud-storage', 'fsspec', 'gcsfs' ], python_version='3', trigger_rule='all_done', dag=dag) Join = DummyOperator(task_id='Join', dag=dag, trigger_rule='all_done') Skip1 = DummyOperator(task_id='Skip1', dag=dag, trigger_rule='all_done') Skip2 = DummyOperator(task_id='Skip2', dag=dag, trigger_rule='all_done') Skip3 = DummyOperator(task_id='Skip3', dag=dag, trigger_rule='all_done') CallDividendApi.set_upstream(CheckCallApi) Skip1.set_upstream(CheckCallApi) CheckCsvLoad.set_upstream(Skip1) CheckCsvLoad.set_upstream(CallDividendApi) CsvLoad.set_upstream(CheckCsvLoad) Skip2.set_upstream(CheckCsvLoad) CheckCalculateProbability.set_upstream(Skip2) CheckCalculateProbability.set_upstream(CsvLoad) CalculateProbability.set_upstream(CheckCalculateProbability) Skip3.set_upstream(CheckCalculateProbability) Join.set_upstream(Skip3) Join.set_upstream(CalculateProbability) #CsvLoad.set_upstream(CallDividendApi) #CalculateProbability.set_upstream(CsvLoad)