def create_prometheus_data_flow(username, project_id, cc_app_id, english_name, dataset): """prometheus 类型的Metric申请数据平台的dataid,并配置默认的清洗入库规则 """ # 数据平台功能没有开启,则直接返回 if not IS_DATA_OPEN: return True, u"数据平台功能暂未开启" # 1. 提交接入部署计划,获取dataid is_ok, data_id = deploy_plan(username, cc_app_id, dataset, DataType.METRIC.value) if not is_ok: notify_manager( u"申请Prometheus Metric dataid[业务ID:%s,项目名:%s]失败,原因:%s,请关注" % (cc_app_id, english_name, data_id)) return False, "申请Prometheus Metric dataid:%s" % data_id # 2. 创建清洗配置,并启动清洗任务 res, result_table_id = setup_clean(username, cc_app_id, data_id, DataType.METRIC.value) if not res: notify_manager(u"创建Prometheus Metric清洗任务失败[%s],原因:%s,请关注" % (english_name, result_table_id)) return False, u"创建Prometheus Metric清洗任务失败:%s" % result_table_id # 3. 创建分发存储,并启动对应的分发任务 res2, msg = setup_shipper(data_id, result_table_id, DataType.METRIC.value) if not res2: notify_manager(u"启动非标准日志采集分发任务失败[%s],原因:%s,请关注" % (project_id, msg)) return False, u"启动非标准日志采集分发任务失败:%s" % msg return True, data_id
def create_prometheus_data_flow(username, project_id, cc_app_id, english_name, dataset): """prometheus 类型的Metric申请数据平台的dataid,并配置默认的清洗入库规则 """ # 数据平台功能没有开启,则直接返回 if not IS_DATA_OPEN: return True, _("数据平台功能暂未开启") # 1. 提交接入部署计划,获取dataid is_ok, data_id = deploy_plan(username, cc_app_id, dataset, DataType.METRIC.value) if not is_ok: message = '''{prefix_msg}Prometheus Metric dataid[{biz}ID:{cc_app_id},{project}:{english_name}]{fail}, {reason}:{data_id},{suffix_msg}'''.format( prefix_msg=_("申请标准日志采集"), biz=_("业务"), cc_app_id=cc_app_id, project=_("项目名"), english_name=english_name, fail=_("失败"), reason=_("原因"), data_id=data_id, suffix_msg=_("失败") ) notify_manager(message) return False, '{}Prometheus Metric dataid:{}'.format(_("申请"), data_id) # 2. 创建清洗配置,并启动清洗任务 res, result_table_id = setup_clean(username, cc_app_id, data_id, DataType.METRIC.value) if not res: notify_manager('''{prefix_msg}Prometheus Metric{clean_task}[{english_name}], {reason}:{table_id},{suffix_msg}'''.format( prefix_msg=_("创建"), clean_task=_("清洗任务失败"), english_name=english_name, reason=_("原因"), table_id=result_table_id, suffix_msg=_("请关注") )) return False, '{}Prometheus Metric{}:{}'.format(_("创建"), _("清洗任务失败"), result_table_id) # 3. 创建分发存储,并启动对应的分发任务 res2, msg = setup_shipper(data_id, result_table_id, DataType.METRIC.value) if not res2: message = '{prefix_msg}[{project_id}],{reason}:{msg},{suffix_msg}'.format( prefix_msg=_("启动非标准日志采集分发任务失败"), project_id=project_id, reason=_("原因"), msg=msg, suffix_msg=_("请关注") ) notify_manager(message) return False, '{}:{}'.format(_("启动非标准日志采集分发任务失败"), msg) return True, data_id
def create_and_start_standard_data_flow(username, project_id, cc_app_id): """ @summary: 标准日志:创建清洗配置,并启动清洗任务;创建分发存储,并启动对应的分发任务 @note: 初始化集群时调用 @return: True/data_project_id, False/error_msg """ # 数据平台功能没有开启,则直接返回 if not IS_DATA_OPEN: return True, _("数据平台功能暂未开启") try: project = ProjectDataInfo.objects.get(project_id=project_id) except Exception: return False, _("请先在数据平台创建项目信息") # db 中已经有任务信息,则说明已经创建/启动了任务,不需要再重复启动 if project.standard_flow_id and project.standard_flow_task_id: return project.standard_flow_id # 已经创建清洗配置则不再重新创建,只分发任务 if not project.standard_flow_id: # 创建清洗配置 res, flow_id = setup_clean(username, cc_app_id, project.standard_data_id, DataType.SLOG.value) if not res: message = '{prefix_msg}[{project_id}],{reason}:{flow_id},{suffix_msg}'.format( prefix_msg=_("启动标准日志采集清洗任务失败"), project_id=project_id, reason=_("原因"), flow_id=flow_id, suffix_msg=_("请关注") ) notify_manager(message) return False, '{}:{}'.format(_("启动标准日志采集清洗任务失败"), flow_id) else: flow_id = project.standard_flow_id # 启动分发任务 res2, flow_task_id = setup_shipper(project.standard_data_id, flow_id, DataType.SLOG.value) if not res2: message = '{prefix_msg}[{project_id}],{reason}:{flow_task_id},{suffix_msg}'.format( prefix_msg=_("启动标准日志采集分发任务失败"), project_id=project_id, reason=_("原因"), flow_task_id=flow_task_id, suffix_msg=_("请关注") ) notify_manager(message) return False, '{}:{}'.format(_("启动标准日志采集清洗任务失败"), flow_task_id) # 将任务相关的id保存到db中,下次初始化集群则可以直接查询状态 project.standard_flow_id = flow_id project.standard_flow_task_id = flow_task_id project.save() return True, _("启动标准日志采集任务成功")
def create_and_start_standard_data_flow(username, project_id, cc_app_id): """ @summary: 标准日志:创建清洗配置,并启动清洗任务;创建分发存储,并启动对应的分发任务 @note: 初始化集群时调用 @return: True/data_project_id, False/error_msg """ # 数据平台功能没有开启,则直接返回 if not IS_DATA_OPEN: return True, u"数据平台功能暂未开启" try: project = ProjectDataInfo.objects.get(project_id=project_id) except Exception: return False, u"请先在数据平台创建项目信息" # db 中已经有任务信息,则说明已经创建/启动了任务,不需要再重复启动 if project.standard_flow_id and project.standard_flow_task_id: return project.standard_flow_id # 已经创建清洗配置则不再重新创建,只分发任务 if not project.standard_flow_id: # 创建清洗配置 res, flow_id = setup_clean(username, cc_app_id, project.standard_data_id, DataType.SLOG.value) if not res: notify_manager(u"启动标准日志采集清洗任务失败[%s],原因:%s,请关注" % (project_id, flow_id)) return False, u"启动标准日志采集清洗任务失败:%s" % flow_id else: flow_id = project.standard_flow_id # 启动分发任务 res2, flow_task_id = setup_shipper(project.standard_data_id, flow_id, DataType.SLOG.value) if not res2: notify_manager(u"启动标准日志采集分发任务失败[%s],原因:%s,请关注" % (project_id, flow_task_id)) return False, u"启动标准日志采集分发任务失败:%s" % flow_task_id # 将任务相关的id保存到db中,下次初始化集群则可以直接查询状态 project.standard_flow_id = flow_id project.standard_flow_task_id = flow_task_id project.save() return True, u"启动标准日志采集任务成功"
def create_prometheus_data_flow(username, project_id, cc_app_id, english_name, dataset): """prometheus 类型的Metric申请数据平台的dataid,并配置默认的清洗入库规则 """ # 数据平台功能没有开启,则直接返回 if not IS_DATA_OPEN: return True, _("数据平台功能暂未开启") # 1. 提交接入部署计划,获取dataid is_ok, data_id = deploy_plan(username, cc_app_id, dataset, DataType.METRIC.value) if not is_ok: message = _( '申请标准日志采集Prometheus Metric dataid[业务ID:{cc_app_id},项目名:{english_name}]失败,原因:{data_id},失败' ).format( cc_app_id=cc_app_id, english_name=english_name, data_id=data_id, ) notify_manager(message) return False, _('申请Prometheus Metric dataid:{}').format(data_id) # 2. 创建清洗配置,并启动清洗任务 res, result_table_id = setup_clean(username, cc_app_id, data_id, DataType.METRIC.value) if not res: notify_manager( _('创建Prometheus Metric清洗任务失败[{english_name}],原因:{table_id}请关注'). format( english_name=english_name, table_id=result_table_id, )) return False, _('创建Prometheus Metric清洗任务失败:{}').format(result_table_id) # 3. 创建分发存储,并启动对应的分发任务 res2, msg = setup_shipper(data_id, result_table_id, DataType.METRIC.value) if not res2: message = _('启动非标准日志采集分发任务失败[{}],原因:{},请关注').format(project_id, msg=msg) notify_manager(message) return False, _('启动非标准日志采集分发任务失败:{}').format(msg) return True, data_id