def test_hive_partition_sensor(self):
op = HivePartitionSensor(
task_id='hive_partition_check',
table='airflow.static_babynames_partitioned',
dag=self.dag)
op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE,
ignore_ti_state=True)
partition = "dt='{}'".format(
chk_d if int(chk_d2) > int(start) else datetime.
strftime(datetime.strptime(start, '%Y%m%d'), '%Y-%m-%d'))
logging.info('Poking for table %s.%s, partition %s',
params.get('hive_db'), params.get('hive_table'),
partition)
from airflow.hooks.hive_hooks import HiveMetastoreHook
hook = HiveMetastoreHook(metastore_conn_id="metastore_default")
return hook.check_for_partition(params.get('hive_db'),
params.get('hive_table'), partition)
table_validate_task = HivePartitionSensor(
task_id="table_validate_task_{}_{}".format(db, table),
table=table,
partition="dt='{{ ds }}'",
schema=db,
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# table_validate_task.pre_execute = get_check_date
table_validate_task.params = table_info
table_validate_task.poke = get_check_date
# 同步hive与mysql表结构
sync_table_schema = PythonOperator(
task_id='sync_table_schema_{}_{}'.format(db, table),
python_callable=init_mysql_table,
provide_context=True,
op_kwargs={
# "conn": hive_cursor,
"db": db,
'email_on_failure': True,
'email_on_retry': False,
}
dag = airflow.DAG(
'dwd_opay_client_event_base_di',
schedule_interval="00 02 * * *",
default_args=args,
)
##----------------------------------------- 依赖 ---------------------------------------##
opay_ep_logv0_prev_hour_task = HivePartitionSensor(
task_id="opay_ep_logv0_prev_hour_task",
table="opay_ep_logv0",
partition="dt='{{ ds }}' and hour='22'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
opay_ep_logv1_prev_hour_task = HivePartitionSensor(
task_id="opay_ep_logv1_prev_hour_task",
table="opay_ep_logv1",
partition="dt='{{ ds }}' and hour='22'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
##----------------------------------------- 任务超时监控 ---------------------------------------##
def fun_task_timeout_monitor(ds, dag, **op_kwargs):
'ofood_dw_ods.ods_sqoop_base_jh_shop_df',
'ofood_dw_ods.ods_sqoop_base_jh_waimai_df',
'ofood_dw_ods.ods_log_client_event_hi'
],
# 任务名称
"task_name": "ofood全局运营指标"
},
dag=dag
)
# 熔断阻塞流程
jh_order_validate_task = HivePartitionSensor(
task_id="jh_order_validate_task",
table="ods_sqoop_base_jh_order_df",
partition="dt='{{ds}}'",
schema="ofood_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
jh_order_log_validate_task = HivePartitionSensor(
task_id="jh_order_log_validate_task",
table="ods_sqoop_base_jh_order_log_df",
partition="dt='{{ds}}'",
schema="ofood_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
# 依赖前一天分区
dependence_dm_oride_driver_base_prev_day_task = UFileSensor(
task_id='dm_oride_driver_base_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dm_oride_driver_base/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖当天分区00点
dependence_server_magic_now_day_task = HivePartitionSensor(
task_id="server_magic_now_day_task",
table="server_magic",
partition="dt='{{macros.ds_add(ds, +1)}}' and hour='00'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖前一天分区
dependence_dwm_oride_driver_finance_di_prev_day_task = UFileSensor(
task_id='dwm_oride_driver_finance_di_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwm_oride_driver_finance_di/country_code=NG",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
db_name = "oride_dw"
table_name = "dwd_oride_order_dispatch_chose_detail_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dependence_dispatch_tracker_server_magic_prev_day_task = HivePartitionSensor(
task_id="dependence_dispatch_tracker_server_magic_prev_day_task",
table="dispatch_tracker_server_magic",
partition="dt='{{ ds }}' and hour='23'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
dependence_dispatch_tracker_server_magic_prev_day_task = HivePartitionSensor(
task_id="dependence_dispatch_tracker_server_magic_prev_day_task",
table="dispatch_tracker_server_magic",
partition="dt='{{ ds }}' and hour='23'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
hdfs_path = "oss://opay-datalake/oride/oride_dw/" + table_name
task_id='dependence_dwd_oride_client_event_detail_hi_prev_day_task',
filepath='{hdfs_path_str}/country_code=nal/dt={pt}/hour={hour}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwd_oride_client_event_detail_hi",
pt='{{ds}}',
hour='23'
),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
# 依赖前一天分区
dependence_dwd_oride_driver_location_event_hi_prev_day_task = HivePartitionSensor(
task_id="dwd_oride_driver_location_event_hi_prev_day_task",
table="ods_log_driver_track_data_hi",
partition="""dt='{{ ds }}' and hour='23'""",
schema="oride_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
dwd_oride_order_base_include_test_di_prev_day_task = S3KeySensor(
task_id='dwd_oride_order_base_include_test_di_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwd_oride_order_base_include_test_di/country_code=NG",
pt='{{ds}}'
),
bucket_name='opay-bi',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
# task_id='sleep_id',
# depends_on_past=False,
# bash_command='sleep 10',
# dag=dag)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_oride_order_trip_raw_feature_hi"
hdfs_path = "oss://opay-datalake/oride/oride_dw/" + table_name
##----------------------------------------- 依赖 ---------------------------------------##
oride_trip_raw_feature_prev_hour_task = HivePartitionSensor(
task_id="oride_trip_raw_feature_prev_hour_task",
table="ods_log_oride_trip_raw_feature_hi",
partition=
"""dt='{{ ds }}' and hour='{{ execution_date.strftime("%H") }}'""",
schema="oride_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
##----------------------------------------- 任务超时监控 ---------------------------------------##
def fun_task_timeout_monitor(ds, execution_date, dag, **op_kwargs):
dag_ids = dag.dag_id
tb = [{
"dag":
dag,
"db":
"oride_dw",
dag = airflow.DAG('dwd_oride_order_skyeye_di',
schedule_interval="30 07 * * *",
default_args=args,
)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_oride_order_skyeye_di"
hdfs_path = "oss://opay-datalake/oride/oride_dw/" + table_name
##----------------------------------------- 依赖 ---------------------------------------##
ods_log_oride_order_skyeye_di_task = HivePartitionSensor(
task_id="dwd_oride_order_skyeye_di_prev_day_task",
table="ods_log_oride_order_skyeye_di",
partition="dt='{{ds}}'",
schema="oride_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
##----------------------------------------- 任务超时监控 ---------------------------------------##
def fun_task_timeout_monitor(ds,dag,**op_kwargs):
dag_ids=dag.dag_id
msg = [
{"dag":dag,"db": "oride_dw", "table":"{dag_name}".format(dag_name=dag_ids), "partition": "country_code=nal/dt={pt}".format(pt=ds), "timeout": "2400"}
]
TaskTimeoutMonitor().set_task_monitor(msg)
validate_partition_data = PythonOperator(
task_id='validate_partition_data',
python_callable=validate_partition,
provide_context=True,
op_kwargs={
# 验证table
"table_names": table_names,
# 任务名称
"task_name": "快车司机档案数据"
},
dag=dag)
data_order_validate_task = HivePartitionSensor(
task_id="data_order_validate_task",
table="ods_sqoop_base_data_order_df",
partition="dt='{{ds}}'",
schema="oride_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
data_driver_comment_validate_task = HivePartitionSensor(
task_id="data_driver_comment_validate_task",
table="ods_sqoop_base_data_driver_comment_df",
partition="dt='{{ds}}'",
schema="oride_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
data_driver_balance_extend_validate_task = HivePartitionSensor(
task_id="data_driver_balance_extend_validate_task",
table="ods_sqoop_base_data_driver_balance_extend_df",
db_name = "oride_dw"
table_name = "dwd_driver_track_data_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dependence_ods_log_driver_track_data_hi_task = HivePartitionSensor(
task_id="dependence_ods_log_driver_track_data_hi_task",
table="ods_log_driver_track_data_hi",
partition="dt='{{ ds }}'",
schema="oride_dw_ods",
poke_interval=60,
dag=dag)
# 路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
# 依赖前一天分区
dependence_ods_log_driver_track_data_hi_task = HivePartitionSensor(
task_id="dependence_ods_log_driver_track_data_hi_task",
table="ods_log_driver_track_data_hi",
partition="dt='{{ ds }}'",
schema="oride_dw_ods",
poke_interval=60,
sleep_time = BashOperator(task_id='sleep_id',
depends_on_past=False,
bash_command='sleep 120',
dag=dag)
sleep_time2 = BashOperator(task_id='sleep_i2',
depends_on_past=False,
bash_command='sleep 20',
dag=dag)
"""
##----依赖数据源---##
"""
dependence_ods_log_oride_order_skyeye_di = HivePartitionSensor(
task_id="dependence_ods_log_oride_order_skyeye_di",
table="ods_log_oride_order_skyeye_di",
partition="dt='{{ ds }}'",
schema="oride_dw_ods",
poke_interval=60,
dag=dag)
dwd_oride_order_base_include_test_di_task = HivePartitionSensor(
task_id='dwd_oride_order_base_include_test_di_task',
table="dwd_oride_order_base_include_test_di",
partition="dt='{{ds}}'",
schema="oride_dw",
poke_interval=60,
dag=dag)
dependence_data_city_conf = HivePartitionSensor(
task_id="dependence_data_city_conf",
table="ods_sqoop_base_data_city_conf_df",
dag = airflow.DAG('app_opos_bd_admin_users_d',
schedule_interval="30 04 * * *",
max_active_runs=1,
default_args=args)
sleep_time = BashOperator(task_id='sleep_id',
depends_on_past=False,
bash_command='sleep 300',
dag=dag)
# 依赖
dependence_table_opos_metrcis_report_task = HivePartitionSensor(
task_id="dependence_table_opos_metrcis_report_task",
table="opos_metrcis_report",
partition="dt='{{ ds }}'",
schema="opos_temp",
poke_interval=120,
dag=dag)
dependence_table_opos_active_user_daily_task = HivePartitionSensor(
task_id="dependence_table_opos_active_user_daily_task",
table="opos_active_user_daily",
partition="dt='{{ ds }}'",
schema="opos_temp",
poke_interval=120,
dag=dag)
# sync_db = Variable.get("app_opos_bd_admin_users").split("\n")
# {"odb":"opay_crm", "otable":"bd_admin_users", "oconn":"opos_opay_crm", "ddb":""}
[
'ofood_dw_ods.ods_sqoop_base_jh_order_df',
'ofood_dw_ods.ods_sqoop_bd_bd_admin_users_df',
'ofood_dw_ods.ods_sqoop_bd_invitation_info_df',
],
# 任务名称
"task_name": "ofood mkt指标"
},
dag=dag
)
# 熔断阻塞流程
jh_order_validate_task = HivePartitionSensor(
task_id="jh_order_validate_task",
table="ods_sqoop_base_jh_order_df",
partition="dt='{{ds}}'",
schema="ofood_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
bd_admin_users_validate_task = HivePartitionSensor(
task_id="bd_admin_users_validate_task",
table="ods_sqoop_bd_bd_admin_users_df",
partition="dt='{{ds}}'",
schema="ofood_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
bd_invitation_info_validate_task = HivePartitionSensor(
task_id="bd_invitation_info_validate_task",
dependence_dwd_oride_order_push_driver_detail_di_prev_day_task = UFileSensor(
task_id='dwd_oride_order_push_driver_detail_di_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_order_push_driver_detail_di/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
# 依赖前一天分区
dependence_oride_driver_timerange_prev_day_task = HivePartitionSensor(
task_id="oride_driver_timerange_prev_day_task",
table="ods_log_oride_driver_timerange",
partition="dt='{{ds}}'",
schema="oride_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dependence_dwd_oride_driver_accept_order_click_detail_di_prev_day_task = UFileSensor(
task_id='dwd_oride_driver_accept_order_click_detail_di_prev_day_task',
filepath='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str=
"oride/oride_dw/dwd_oride_driver_accept_order_click_detail_di/country_code=nal",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dependence_dwd_oride_driver_accept_order_show_detail_di_prev_day_task = UFileSensor(
def tesk_dependence(self, tables, dag):
dependence = []
try:
for item in tables:
#读取 db、table、partition
table = item.get('table', None)
db = item.get('db', None)
partition = item.get('partitions', None)
if table is None or db is None or partition is None:
return None
#读取hive location地址
location = self.get_hive_location(db, table)
#替换原有bucket
location = location.replace('ufile://opay-datalake/', '')
#task_id 名称
task_id_flag = table + "_task"
#区分ods的依赖路径
if db[-3:].lower() == 'ods' or db[-2:].lower() == 'bi':
# 配置依赖关系(前一天分区)
dependence_task_flag = HivePartitionSensor(
task_id='dependence_{task_id_name}'.format(
task_id_name=task_id_flag),
table=table,
partition="dt='{{ds}}'",
schema=db,
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
else:
# 配置依赖关系(前一天分区)
dependence_task_flag = UFileSensor(
task_id='dependence_{task_id_name}'.format(
task_id_name=task_id_flag),
filepath=
'{hdfs_path_name}/{partition_name}/dt={{{{ds}}}}/_SUCCESS'
.format(hdfs_path_name=location,
partition_name=partition),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
dependence.append(dependence_task_flag)
return dependence
except Exception as e:
#self.comwx.postAppMessage('DW调度系统任务 {jobname} 任务依赖列表产出异常'.format(jobname=dag.dag_id),'271')
logging.info(e)
sys.exit(1)
'email_on_failure': True,
'email_on_retry': False,
}
dag = airflow.DAG('app_opay_user_device_d',
schedule_interval="00 03 * * *",
default_args=args,
)
##----------------------------------------- 依赖 ---------------------------------------##
# 依赖前一天分区
dwd_opay_client_event_base_di_prev_day_task = HivePartitionSensor(
task_id="dwd_opay_client_event_base_di_prev_day_task",
table="dwd_opay_client_event_base_di",
partition="dt='{{ ds }}'",
schema="opay_dw",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
app_opay_user_device_d_prev_day_task = OssSensor(
task_id='app_opay_user_device_d_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="opay/opay_dw/app_opay_user_device_d/country_code=nal",
pt='{{macros.ds_add(ds, -1)}}'
),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
#判断ufile(cdh环境)
if code_map["id"].lower()=="ufile":
dependence_dwm_oride_order_base_di_task = UFileSensor(
task_id='dwm_oride_order_base_di_task',
filepath='{hdfs_path_str}/country_code=nal/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dwm_oride_order_base_di",
pt='{{ds}}'
),
bucket_name='opay-datalake',
poke_interval=60,
dag=dag
)
dependence_dim_oride_city_task = HivePartitionSensor(
task_id="dim_oride_city_task",
table="dim_oride_city",
partition="dt='{{ds}}'",
schema="oride_dw",
poke_interval=60, # 依赖 不满足时，一分钟检查一次依赖状态
dag=dag
)
dependence_dim_oride_passenger_base_task = HivePartitionSensor(
task_id="dim_oride_passenger_base_task",
table="dim_oride_passenger_base",
partition="dt='{{ds}}'",
schema="oride_dw",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
dependence_dim_oride_driver_base_task = UFileSensor(
task_id='dim_oride_driver_base_task',
filepath='{hdfs_path_str}/country_code=NG/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride/oride_dw/dim_oride_driver_base",
db_name="oride_dw"
table_name="dwd_oride_anti_fraud_log_di"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map=eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower()=="ufile":
#依赖前一天分区
log_anti_oride_fraud_task=HivePartitionSensor(
task_id="log_anti_oride_fraud_task",
table="log_anti_oride_fraud",
partition="dt='{{macros.ds_add(ds, +1)}}' and hour='00'",
schema="oride_source",
poke_interval=60, #依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
# 路径
hdfs_path="ufile://opay-datalake/oride/oride_dw/dwd_oride_anti_fraud_log_new_di"
else:
print("成功")
# 依赖前一天分区
log_anti_oride_fraud_task = HivePartitionSensor(
task_id="log_anti_oride_fraud_task",
table="log_anti_oride_fraud",
partition="dt='{{macros.ds_add(ds, +1)}}' and hour='00'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag = airflow.DAG('dwd_oride_driver_cheating_detection_hi',
schedule_interval="30 * * * *",
default_args=args)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_oride_driver_cheating_detection_hi"
hdfs_path = "oss://opay-datalake/oride/oride_dw/" + table_name
##----------------------------------------- 依赖 ---------------------------------------##
server_event_task = HivePartitionSensor(
task_id="server_event_task",
table="server_event",
partition=
"""dt='{{ ds }}' and hour='{{ execution_date.strftime("%H") }}'""",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
##----------------------------------------- 任务超时监控 ---------------------------------------##
def fun_task_timeout_monitor(ds, dag, execution_date, **op_kwargs):
dag_ids = dag.dag_id
tb = [{
"dag":
dag,
"db":
"oride_dw",
db_name="oride_dw"
table_name="dwd_oride_client_event_detail_hi"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map=eval(Variable.get("sys_flag"))
# 依赖前一小时分区
client_event_prev_hour_task = HivePartitionSensor(
task_id="client_event_prev_hour_task",
table="client_event",
partition="""dt='{{ ds }}' and hour='{{ execution_date.strftime("%H") }}'""",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
opay_ep_logv0_prev_hour_task = HivePartitionSensor(
task_id="opay_ep_logv0_prev_hour_task",
table="opay_ep_logv0",
partition="""dt='{{ ds }}' and hour='{{ execution_date.strftime("%H") }}'""",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag
)
opay_ep_logv1_prev_hour_task = HivePartitionSensor(
'ofood_dw_ods.ods_sqoop_bd_invitation_info_df',
'ofood_dw_ods.ods_sqoop_bd_jh_member_df',
'ofood_dw_ods.ods_sqoop_base_jh_shop_account_df',
'ofood_dw_ods.ods_sqoop_base_jh_waimai_product_df',
],
# 任务名称
"task_name":
"ofood BD指标"
},
dag=dag)
# 熔断阻塞流程
jh_order_validate_task = HivePartitionSensor(
task_id="jh_order_validate_task",
table="ods_sqoop_base_jh_order_df",
partition="dt='{{ds}}'",
schema="ofood_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
jh_order_time_validate_task = HivePartitionSensor(
task_id="jh_order_time_validate_task",
table="ods_sqoop_base_jh_order_time_df",
partition="dt='{{ds}}'",
schema="ofood_dw_ods",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
jh_waimai_comment_validate_task = HivePartitionSensor(
task_id="jh_waimai_comment_validate_task",
table="ods_sqoop_base_jh_waimai_comment_df",
count(distinct user_id) as users,
sum(price) as price_total,
sum(reward) as reward_total
FROM
oride_source.user_order
WHERE
dt='{{ ds }}'
GROUP BY status
""",
schema='dashboard',
dag=dag)
dependent_dwd_oride_driver_timerange_di = HivePartitionSensor(
task_id="dependent_dwd_oride_driver_timerange_di",
table="dwd_oride_driver_timerange_di",
schema="oride_dw",
partition="dt='{{ds}}'",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
create_oride_driver_daily_summary = HiveOperator(
task_id='create_oride_driver_daily_summary',
hql="""
CREATE TABLE IF NOT EXISTS oride_driver_daily_summary (
driver_id bigint,
phone_number string,
real_name string,
group_id int,
group_name string,
group_leader string,
online_time bigint,
schedule_interval="30 01 * * *",
default_args=args)
##----------------------------------------- 变量 ---------------------------------------##
db_name = "oride_dw"
table_name = "app_oride_driver_gps_info"
hdfs_path = "oss://opay-datalake/oride/oride_dw/" + table_name
##----------------------------------------- 依赖 ---------------------------------------##
# 依赖前一天分区
moto_locations_prev_day_task = HivePartitionSensor(
task_id="moto_locations_prev_day_task",
table="moto_locations",
partition="dt='{{ds}}'",
schema="oride_source",
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
ods_sqoop_base_oride_assets_sku_df_prev_day_task = OssSensor(
task_id='ods_sqoop_base_oride_assets_sku_df_prev_day_task',
bucket_key='{hdfs_path_str}/dt={pt}/_SUCCESS'.format(
hdfs_path_str="oride_dw_sqoop/opay_assets/oride_assets_sku",
pt='{{ds}}'),
bucket_name='opay-datalake',
poke_interval=60, # 依赖不满足时，一分钟检查一次依赖状态
dag=dag)
##----------------------------------------- 任务超时监控 ---------------------------------------##
db_name = "oride_dw"
table_name = "dwd_driver_track_data_hi"
##----------------------------------------- 依赖 ---------------------------------------##
#获取变量
code_map = eval(Variable.get("sys_flag"))
#判断ufile(cdh环境)
if code_map["id"].lower() == "ufile":
# 依赖前一天分区
dependence_ods_log_driver_track_data_hi_task = HivePartitionSensor(
task_id="dependence_ods_log_driver_track_data_hi_task",
table="ods_log_driver_track_data_hi",
partition=
""" dt='{{ ds }}' and hour='{{ execution_date.strftime("%H") }}' """,
schema="oride_dw_ods",
poke_interval=60,
dag=dag)
# 路径
hdfs_path = "ufile://opay-datalake/oride/oride_dw/" + table_name
else:
print("成功")
# 依赖前一天分区
dependence_ods_log_driver_track_data_hi_task = HivePartitionSensor(
task_id="dependence_ods_log_driver_track_data_hi_task",
table="ods_log_driver_track_data_hi",
partition=
""" dt='{{ ds }}' and hour='{{ execution_date.strftime("%H") }}' """,
schema="oride_dw_ods",