##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_oride_rider_signups_df"
##----------------------------------------- 依赖 ---------------------------------------##
# 获取变量
code_map = eval(Variable.get("sys_flag"))
# 判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
ods_sqoop_mass_rider_signups_df_tesk = UFileSensor(
task_id='ods_sqoop_mass_rider_signups_df_tesk',
filepath="{hdfs_path_str}/dt={pt}/_SUCCESS".format(
hdfs_path_str="oride_dw_sqoop/opay_spread/rider_signups",
pt="{{ds}}"),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/dwd_oride_rider_signups_df"
else:
ods_sqoop_mass_rider_signups_df_tesk = OssSensor(
task_id='ods_sqoop_mass_rider_signups_df_tesk',
bucket_key="{hdfs_path_str}/dt={pt}/_SUCCESS".format(
hdfs_path_str="oride_dw_sqoop/opay_spread/rider_signups",
pt="{{ds}}"),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 路径
db_name = "oride_dw"
table_name = "dwd_oride_weather_per_10min_df"
##----------------------------------------- 依赖 ---------------------------------------##
# 获取变量
code_map = eval(Variable.get("sys_flag"))
# 判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
ods_sqoop_base_weather_per_10min_df_task = UFileSensor(
task_id='ods_sqoop_base_weather_per_10min_df_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride_dw_sqoop/bi/weather_per_10min", pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
ods_sqoop_base_weather_per_10min_df_task = OssSensor(
task_id='ods_sqoop_base_weather_per_10min_df_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride_dw_sqoop/bi/weather_per_10min", pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_oride_abnormal_order_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
ods_sqoop_base_data_abnormal_order_df_task = UFileSensor(
task_id='ods_sqoop_base_data_abnormal_order_df_tesk',
filepath="{hdfs_path_str}/dt={pt}/_SUCCESS".format(
hdfs_path_str="oride_dw_sqoop/oride_data/data_abnormal_order",
pt="{{ds}}"),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
ods_sqoop_base_data_abnormal_order_df_task = OssSensor(
task_id='ods_sqoop_base_data_abnormal_order_df_tesk',
bucket_key="{hdfs_path_str}/dt={pt}/_SUCCESS".format(
hdfs_path_str="oride_dw_sqoop/oride_data/data_abnormal_order",
pt="{{ds}}"),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
db_name = "oride_dw"
table_name = "dwd_oride_driver_accept_order_show_detail_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dwd_oride_driver_accept_order_show_detail_di_prev_day_task = UFileSensor(
task_id='dwd_oride_driver_accept_order_show_detail_di_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/hour={hour}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_client_event_detail_hi/country_code=nal",
pt='{{ds}}',
hour='23'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
# 依赖前一天分区
dwd_oride_driver_accept_order_show_detail_di_prev_day_task = OssSensor(
task_id='dwd_oride_driver_accept_order_show_detail_di_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/hour={hour}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_client_event_detail_hi/country_code=nal",
pt='{{ds}}',
db_name = "oride_dw"
table_name = "app_oride_driver_transport_capacity_d"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map=eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower()=="ufile":
# 依赖前一天分区
dwm_oride_driver_base_df_prev_day_task = UFileSensor(
task_id='dwm_oride_driver_base_df_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwm_oride_driver_base_df/country_code=NG",
pt='{{ds}}'
),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
dim_oride_city_task = UFileSensor(
task_id='dim_oride_city_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dim_oride_city/country_code=NG",
pt='{{ds}}'
),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
global_table_names = [
'oride_dw.app_oride_global_operate_report_d',
'oride_dw_ods.ods_sqoop_base_data_city_conf_df',
]
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
app_oride_global_operate_report_d_task = UFileSensor(
task_id='app_oride_global_operate_report_d_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/app_oride_global_operate_report_d/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖前一天分区
app_oride_global_operate_report_multi_d_task = UFileSensor(
task_id='app_oride_global_operate_report_multi_d_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/app_oride_global_operate_report_multi_d/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
db_name = "oride_dw"
table_name = "dwm_oride_passenger_base_df"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dim_oride_passenger_base_prev_day_task = UFileSensor(
task_id='dim_oride_passenger_base_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dim_oride_passenger_base/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dwd_oride_order_base_include_test_di_prev_day_task = S3KeySensor(
task_id='dwd_oride_order_base_include_test_di_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_base_include_test_di/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-bi',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
db_name = "oride_dw"
table_name = "app_oride_global_operate_report_d"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dependence_dm_oride_order_base_d_prev_day_task = UFileSensor(
task_id='dm_oride_order_base_d_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dm_oride_order_base_d/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖前一天分区
dependence_dm_oride_passenger_base_cube_prev_day_task = UFileSensor(
task_id='dm_oride_passenger_base_cube_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dm_oride_passenger_base_cube/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
db_name = "oride_dw"
table_name = "app_oride_client_advertising_d"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖 ufile://opay-datalake/oride/oride_dw/dwd_oride_client_event_detail_hi/country_code=nal/dt=2019-08-09/hour=06'
dependence_dwd_oride_client_event_detail_hi = UFileSensor(
task_id="dependence_dwd_oride_client_event_detail_hi",
filepath='{hdfs_path_str}/dt={pt}/hour=23'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_client_event_detail_hi/country_code=nal",
pt='{{ ds }}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
#路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
dependence_dwd_oride_client_event_detail_hi = OssSensor(
task_id="dependence_dwd_oride_client_event_detail_hi",
bucket_key='{hdfs_path_str}/dt={pt}/hour=23/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_client_event_detail_hi/country_code=nal",
pt='{{ ds }}'),
bucket_name='opay-datalake',
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "app_oride_driver_group_d"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
dim_oride_driver_base_task = UFileSensor(
task_id='dim_oride_driver_base_task',
filepath='{hdfs_path_str}/country_code=nal/dt={pt}/_SUCCESS'.format(
hdfs_path_str='oride/oride_dw/dim_oride_driver_base', pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60,
dag=dag)
dwd_oride_order_base_include_test_di_task = S3KeySensor(
task_id='dwd_oride_order_base_include_test_di_task',
bucket_key='{hdfs_path_str}/country_code=NG/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwd_oride_order_base_include_test_di",
pt='{{ds}}'),
bucket_name='opay-bi',
poke_interval=60,
dag=dag)
dwd_oride_driver_data_group_df_task = UFileSensor(
task_id='dwd_oride_driver_data_group_df_task',
# 依赖前一天分区
dependence_dwd_oride_order_base_include_test_di_prev_day_task = S3KeySensor(
task_id='dwd_oride_order_base_include_test_di_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_base_include_test_di/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-bi',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dependence_dwd_oride_order_skyeye_di_prev_day_task = UFileSensor(
task_id='dwd_oride_order_skyeye_di_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_skyeye_di/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
dependence_dwd_oride_order_base_include_test_di_prev_day_task = OssSensor(
task_id='dwd_oride_order_base_include_test_di_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_base_include_test_di/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-datalake',
task_id='dependence_dwd_oride_order_base_include_test_di_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_base_include_test_di/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-bi',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖hive表分区
dwd_oride_driver_cheating_detection_hi_task = UFileSensor(
task_id="dwd_oride_driver_cheating_detection_hi_task",
filepath='{hdfs_path_str}/dt={pt}/hour=23'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_driver_cheating_detection_hi/country_code=nal",
pt='{{ ds }}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
ods_sqoop_promoter_promoter_user_df_task = UFileSensor(
task_id='ods_sqoop_promoter_promoter_user_df_tesk',
filepath="{hdfs_path_str}/dt={pt}/_SUCCESS".format(
hdfs_path_str="oride_dw_sqoop/opay_spread/promoter_user",
pt="{{ds}}"),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
ods_sqoop_mass_rider_signups_df_task = UFileSensor(
db_name = "oride_dw"
table_name = "dm_oride_driver_base_cube_d"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dependence_dim_oride_driver_audit_base_prev_day_task = UFileSensor(
task_id='dim_oride_driver_audit_base_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dim_oride_driver_audit_base/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dependence_dwd_oride_order_base_include_test_di_prev_day_task = S3KeySensor(
task_id='dwd_oride_order_base_include_test_di_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_base_include_test_di/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-bi',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
#获取变量
code_map=eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower()=="ufile":
# 依赖前一天分区
dwd_oride_driver_cheating_detection_hi_prev_hour_task = UFileSensor(
task_id='dwd_oride_driver_cheating_detection_hi_prev_hour_task',
filepath='{hdfs_path_str}/dt={pt}/hour={hour}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwd_oride_driver_cheating_detection_hi/country_code=nal",
pt='{{ds}}',
hour='{{ execution_date.strftime("%H") }}'
),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
else:
dwd_oride_driver_cheating_detection_hi_prev_hour_task = OssSensor(
task_id='dwd_oride_driver_cheating_detection_hi_prev_hour_task',
bucket_key='{hdfs_path_str}/dt={pt}/hour={hour}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwd_oride_driver_cheating_detection_hi/country_code=nal",
pt='{{ds}}',
hour='{{ execution_date.strftime("%H") }}'
),
bucket_name='opay-datalake',
##----------------------------------------- 依赖 ---------------------------------------##
#ufile://opay-datalake/oride_dw_sqoop/anti_fraud_oride/driver_virtual_device_login
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
ods_sqoop_base_driver_virtual_device_login_df_task = UFileSensor(
task_id='ods_sqoop_base_driver_virtual_device_login_df_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride_dw_sqoop/anti_fraud_oride/driver_virtual_device_login",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
ods_sqoop_base_driver_virtual_device_login_df_task = OssSensor(
task_id='ods_sqoop_base_driver_virtual_device_login_df_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride_dw_sqoop/anti_fraud_oride/driver_virtual_device_login",
pt='{{ds}}'),
bucket_name='opay-datalake',
db_name = "oride_dw"
table_name = "dwd_oride_finance_driver_repayment_extend_df"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
ods_sqoop_base_data_driver_balance_extend_df_prev_day_tesk = UFileSensor(
task_id='ods_sqoop_base_data_driver_balance_extend_df_prev_day_tesk',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride_dw_sqoop/oride_data/data_driver_balance_extend",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖前一天分区
ods_sqoop_base_data_driver_records_day_df_prev_day_tesk = UFileSensor(
task_id="ods_sqoop_base_data_driver_records_day_df_prev_day_tesk",
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride_dw_sqoop/oride_data/data_driver_records_day",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
schedule_interval="30 03 * * 1",
default_args=args,
)
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map=eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower()=="ufile":
dwm_oride_order_base_di_task = UFileSensor(
task_id='dwm_oride_order_base_di_task',
filepath='{hdfs_path_str}/country_code=NG/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwm_oride_order_base_di",
pt='{{ds}}'
),
bucket_name='opay-datalake',
poke_interval=60,
dag=dag
)
dwm_oride_passenger_base_df_task = UFileSensor(
task_id='dwm_oride_passenger_base_df_task',
filepath='{hdfs_path_str}/country_code=nal/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwm_oride_passenger_base_df",
pt='{{macros.ds_add(ds, +6)}}'
),
bucket_name='opay-datalake',
poke_interval=60,
dag=dag
)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dim_oride_driver_whitelist_base"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
ods_sqoop_base_data_driver_whitelist_df_task = UFileSensor(
task_id='ods_sqoop_base_data_driver_whitelist_df_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride_dw_sqoop/oride_data/data_driver_whitelist",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
#路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
ods_sqoop_base_data_driver_whitelist_df_task = OssSensor(
task_id='ods_sqoop_base_data_driver_whitelist_df_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride_dw_sqoop/oride_data/data_driver_whitelist",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_oride_driver_timerange_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
ods_log_oride_driver_timerange_task = UFileSensor(
task_id='ods_log_oride_driver_timerange_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw_ods/ods_log_oride_driver_timerange",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
ods_log_oride_driver_timerange_task = OssSensor(
task_id='ods_log_oride_driver_timerange_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw_ods/ods_log_oride_driver_timerange",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
db_name = "oride_dw"
table_name = "dwd_oride_passanger_location_event_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一小时分区
dwd_oride_passanger_location_event_hi_prev_day_task = UFileSensor(
task_id='dwd_oride_passanger_location_event_hi_prev_day_task',
filepath='{hdfs_path_str}/country_code=nal/dt={pt}/hour={hour}/_SUCCESS'
.format(hdfs_path_str=
"oride/oride_dw/dwd_oride_passanger_location_event_hi",
pt='{{ds}}',
hour='23'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
dwd_oride_passanger_location_event_hi_prev_day_task = OssSensor(
task_id='dwd_oride_passanger_location_event_hi_prev_day_task',
bucket_key=
'{hdfs_path_str}/country_code=nal/dt={pt}/hour={hour}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_passanger_location_event_hi",
pt='{{ds}}',
hour='23'),
db_name = "oride_dw"
table_name = "app_oride_finance_driver_repayment_test_d"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天数据是否存在
dim_oride_driver_base_prev_day_task = UFileSensor(
task_id='dim_oride_driver_base_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dim_oride_driver_base/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dwd_oride_finance_driver_repayment_extend_df_tesk = UFileSensor(
task_id='dwd_oride_finance_driver_repayment_extend_df_tesk',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_finance_driver_repayment_extend_df/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)