def _create_and_start_data_flow(username, project_id, cc_biz_id, data_type):
if not IS_DATA_OPEN:
return True, _("数据平台功能暂未开启")
try:
project_data = ProjectDataInfo.objects.get(project_id=project_id, cc_biz_id=cc_biz_id)
except ProjectDataInfo.DoesNotExist:
return False, _("请先在数据平台创建项目信息")
if not project_data.standard_data_id and data_type == DataType.SLOG.value:
logger.error("no standard_data_id")
return True, ""
if not project_data.non_standard_data_id and data_type == DataType.CLOG.value:
logger.error("no non_standard_data_id")
return True, ""
data_bus_cls = create_data_bus(data_type)
data_bus = data_bus_cls(project_data)
result, message = data_bus.clean_and_storage_data(username)
if not result:
message = _("启动{}日志采集清洗和入库任务失败[{}],原因:{},请关注").format(data_type, project_id, message)
notify_manager(message)
return False, message
return True, message
def create_and_start_standard_data_flow(username, project_id, cc_app_id):
"""
@summary: 标准日志：创建清洗配置,并启动清洗任务;创建分发存储，并启动对应的分发任务
@note: 初始化集群时调用
@return: True/data_project_id, False/error_msg
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True, _("数据平台功能暂未开启")
try:
project = ProjectDataInfo.objects.get(project_id=project_id)
except Exception:
return False, _("请先在数据平台创建项目信息")
# db 中已经有任务信息，则说明已经创建/启动了任务，不需要再重复启动
if project.standard_flow_id and project.standard_flow_task_id:
return project.standard_flow_id
# 已经创建清洗配置则不再重新创建，只分发任务
if not project.standard_flow_id:
# 创建清洗配置
res, flow_id = setup_clean(username, cc_app_id, project.standard_data_id, DataType.SLOG.value)
if not res:
message = '{prefix_msg}[{project_id}],{reason}:{flow_id},{suffix_msg}'.format(
prefix_msg=_("启动标准日志采集清洗任务失败"),
project_id=project_id,
reason=_("原因"),
flow_id=flow_id,
suffix_msg=_("请关注")
)
notify_manager(message)
return False, '{}:{}'.format(_("启动标准日志采集清洗任务失败"), flow_id)
else:
flow_id = project.standard_flow_id
# 启动分发任务
res2, flow_task_id = setup_shipper(project.standard_data_id, flow_id, DataType.SLOG.value)
if not res2:
message = '{prefix_msg}[{project_id}],{reason}:{flow_task_id},{suffix_msg}'.format(
prefix_msg=_("启动标准日志采集分发任务失败"),
project_id=project_id,
reason=_("原因"),
flow_task_id=flow_task_id,
suffix_msg=_("请关注")
)
notify_manager(message)
return False, '{}:{}'.format(_("启动标准日志采集清洗任务失败"), flow_task_id)
# 将任务相关的id保存到db中，下次初始化集群则可以直接查询状态
project.standard_flow_id = flow_id
project.standard_flow_task_id = flow_task_id
project.save()
return True, _("启动标准日志采集任务成功")
def create_data_project(username, project_id, cc_app_id, english_name):
"""
@summary: 在数据平台上创建项目信息
@note: 创建项目时调用，关联了蓝鲸业务的项目才需要创建
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True
project, _c = ProjectDataInfo.objects.get_or_create(
project_id=project_id, defaults={'cc_biz_id': cc_app_id})
# 判断项目是否已经创建，已经创建则不再重复创建
if all([
project.data_project_id, project.standard_data_id,
project.non_standard_data_id
]):
return True
# 申请标准日志采集 dataid, standard_data_name 修改 db 中的字段信息
standard_data_name = f'{DataType.SLOG.value}_{english_name}'
res1, standard_data_id = deploy_plan(username, cc_app_id,
standard_data_name,
DataType.SLOG.value)
if not res1:
notify_manager(u"申请标准日志采集dataid[业务ID:%s,项目名:%s]失败,原因:%s,请关注" %
(cc_app_id, english_name, standard_data_id))
raise error_codes.APIError.f("申请标准日志采集dataid失败:%s" % standard_data_id)
# 申请非标准日志采集 dataid
non_standard_data_name = f'{DataType.CLOG.value}_{english_name}'
res2, non_standard_data_id = deploy_plan(username, cc_app_id,
non_standard_data_name,
DataType.CLOG.value)
if not res2:
notify_manager(u"申请非标准日志采集dataid[业务ID:%s,项目名:%s]失败,原因:%s,请关注" %
(cc_app_id, english_name, non_standard_data_id))
raise error_codes.APIError.f("申请非标准日志采集dataid失败:%s" %
non_standard_data_id)
# 数据平台 V3 API 没有project_id的概念，给一个默认的值
project.data_project_id = 1
project.standard_data_id = standard_data_id
project.standard_data_name = standard_data_name
project.non_standard_data_id = non_standard_data_id
project.non_standard_data_name = non_standard_data_name
project.save()
return True
def deploy_log_plan(username, cc_biz_id, project_code, data_log_type):
data_log_name = f"{data_log_type}_{project_code}_{cc_biz_id}"
data_id = get_data_id_by_name(data_log_name)
if data_id:
return data_id, data_log_name
result, data = deploy_plan(username, cc_biz_id, data_log_name, data_log_type)
if not result:
notify_manager(
_("申请{data_log_type}采集dataid[业务ID:{cc_biz_id},项目名:{project_code}]失败,原因:{data},请关注").format(
data_log_type=data_log_type, cc_biz_id=cc_biz_id, project_code=project_code, data=data
)
)
raise error_codes.APIError(_("申请{}采集dataid失败:{}").format(data_log_type, data))
return data, data_log_name
def create_and_start_standard_data_flow(username, project_id, cc_app_id):
"""
@summary: 标准日志：创建清洗配置,并启动清洗任务;创建分发存储，并启动对应的分发任务
@note: 初始化集群时调用
@return: True/data_project_id, False/error_msg
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True, u"数据平台功能暂未开启"
try:
project = ProjectDataInfo.objects.get(project_id=project_id)
except Exception:
return False, u"请先在数据平台创建项目信息"
# db 中已经有任务信息，则说明已经创建/启动了任务，不需要再重复启动
if project.standard_flow_id and project.standard_flow_task_id:
return project.standard_flow_id
# 已经创建清洗配置则不再重新创建，只分发任务
if not project.standard_flow_id:
# 创建清洗配置
res, flow_id = setup_clean(username, cc_app_id,
project.standard_data_id,
DataType.SLOG.value)
if not res:
notify_manager(u"启动标准日志采集清洗任务失败[%s],原因:%s,请关注" %
(project_id, flow_id))
return False, u"启动标准日志采集清洗任务失败:%s" % flow_id
else:
flow_id = project.standard_flow_id
# 启动分发任务
res2, flow_task_id = setup_shipper(project.standard_data_id, flow_id,
DataType.SLOG.value)
if not res2:
notify_manager(u"启动标准日志采集分发任务失败[%s],原因:%s,请关注" %
(project_id, flow_task_id))
return False, u"启动标准日志采集分发任务失败:%s" % flow_task_id
# 将任务相关的id保存到db中，下次初始化集群则可以直接查询状态
project.standard_flow_id = flow_id
project.standard_flow_task_id = flow_task_id
project.save()
return True, u"启动标准日志采集任务成功"
def create_prometheus_data_flow(username, project_id, cc_app_id, english_name,
dataset):
"""prometheus 类型的Metric申请数据平台的dataid，并配置默认的清洗入库规则
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True, u"数据平台功能暂未开启"
# 1. 提交接入部署计划,获取dataid
is_ok, data_id = deploy_plan(username, cc_app_id, dataset,
DataType.METRIC.value)
if not is_ok:
notify_manager(
u"申请Prometheus Metric dataid[业务ID:%s,项目名:%s]失败,原因:%s,请关注" %
(cc_app_id, english_name, data_id))
return False, "申请Prometheus Metric dataid:%s" % data_id
# 2. 创建清洗配置,并启动清洗任务
res, result_table_id = setup_clean(username, cc_app_id, data_id,
DataType.METRIC.value)
if not res:
notify_manager(u"创建Prometheus Metric清洗任务失败[%s],原因:%s,请关注" %
(english_name, result_table_id))
return False, u"创建Prometheus Metric清洗任务失败:%s" % result_table_id
# 3. 创建分发存储，并启动对应的分发任务
res2, msg = setup_shipper(data_id, result_table_id, DataType.METRIC.value)
if not res2:
notify_manager(u"启动非标准日志采集分发任务失败[%s],原因:%s,请关注" % (project_id, msg))
return False, u"启动非标准日志采集分发任务失败:%s" % msg
return True, data_id
def create_prometheus_data_flow(username, project_id, cc_app_id, english_name, dataset):
"""prometheus 类型的Metric申请数据平台的dataid，并配置默认的清洗入库规则
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True, _("数据平台功能暂未开启")
# 1. 提交接入部署计划,获取dataid
is_ok, data_id = deploy_plan(username, cc_app_id, dataset, DataType.METRIC.value)
if not is_ok:
message = '''{prefix_msg}Prometheus Metric dataid[{biz}ID:{cc_app_id},{project}:{english_name}]{fail},
{reason}:{data_id},{suffix_msg}'''.format(
prefix_msg=_("申请标准日志采集"),
biz=_("业务"),
cc_app_id=cc_app_id,
project=_("项目名"),
english_name=english_name,
fail=_("失败"),
reason=_("原因"),
data_id=data_id,
suffix_msg=_("失败")
)
notify_manager(message)
return False, '{}Prometheus Metric dataid:{}'.format(_("申请"), data_id)
# 2. 创建清洗配置,并启动清洗任务
res, result_table_id = setup_clean(username, cc_app_id, data_id, DataType.METRIC.value)
if not res:
notify_manager('''{prefix_msg}Prometheus Metric{clean_task}[{english_name}],
{reason}:{table_id},{suffix_msg}'''.format(
prefix_msg=_("创建"),
clean_task=_("清洗任务失败"),
english_name=english_name,
reason=_("原因"),
table_id=result_table_id,
suffix_msg=_("请关注")
))
return False, '{}Prometheus Metric{}:{}'.format(_("创建"), _("清洗任务失败"), result_table_id)
# 3. 创建分发存储，并启动对应的分发任务
res2, msg = setup_shipper(data_id, result_table_id, DataType.METRIC.value)
if not res2:
message = '{prefix_msg}[{project_id}],{reason}:{msg},{suffix_msg}'.format(
prefix_msg=_("启动非标准日志采集分发任务失败"),
project_id=project_id,
reason=_("原因"),
msg=msg,
suffix_msg=_("请关注")
)
notify_manager(message)
return False, '{}:{}'.format(_("启动非标准日志采集分发任务失败"), msg)
return True, data_id
def create_prometheus_data_flow(username, project_id, cc_app_id, english_name,
dataset):
"""prometheus 类型的Metric申请数据平台的dataid，并配置默认的清洗入库规则
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True, _("数据平台功能暂未开启")
# 1. 提交接入部署计划,获取dataid
is_ok, data_id = deploy_plan(username, cc_app_id, dataset,
DataType.METRIC.value)
if not is_ok:
message = _(
'申请标准日志采集Prometheus Metric dataid[业务ID:{cc_app_id},项目名:{english_name}]失败,原因:{data_id},失败'
).format(
cc_app_id=cc_app_id,
english_name=english_name,
data_id=data_id,
)
notify_manager(message)
return False, _('申请Prometheus Metric dataid:{}').format(data_id)
# 2. 创建清洗配置,并启动清洗任务
res, result_table_id = setup_clean(username, cc_app_id, data_id,
DataType.METRIC.value)
if not res:
notify_manager(
_('创建Prometheus Metric清洗任务失败[{english_name}],原因:{table_id}请关注').
format(
english_name=english_name,
table_id=result_table_id,
))
return False, _('创建Prometheus Metric清洗任务失败:{}').format(result_table_id)
# 3. 创建分发存储，并启动对应的分发任务
res2, msg = setup_shipper(data_id, result_table_id, DataType.METRIC.value)
if not res2:
message = _('启动非标准日志采集分发任务失败[{}],原因:{},请关注').format(project_id,
msg=msg)
notify_manager(message)
return False, _('启动非标准日志采集分发任务失败:{}').format(msg)
return True, data_id
def create_data_project(username, project_id, cc_app_id, english_name):
"""
@summary: 在数据平台上创建项目信息
@note: 创建项目时调用，关联了蓝鲸业务的项目才需要创建
"""
# 数据平台功能没有开启，则直接返回
if not IS_DATA_OPEN:
return True
project, _c = ProjectDataInfo.objects.get_or_create(project_id=project_id, defaults={'cc_biz_id': cc_app_id})
# 判断项目是否已经创建，已经创建则不再重复创建
if all([project.data_project_id, project.standard_data_id, project.non_standard_data_id]):
return True
# 申请标准日志采集 dataid, standard_data_name 修改 db 中的字段信息
standard_data_name = f'{DataType.SLOG.value}_{english_name}'
res1, standard_data_id = deploy_plan(username, cc_app_id, standard_data_name, DataType.SLOG.value)
if not res1:
message = '''{prefix_msg}dataid{biz}ID:{cc_app_id},{project}:{english_name}{fail},
{reason}:{standard_data_id},{suffix_msg}'''.format(
prefix_msg=_("申请标准日志采集"),
biz=_("业务"),
cc_app_id=cc_app_id,
project=_("项目名"),
english_name=english_name,
fail=_("失败"),
reason=_("原因"),
standard_data_id=standard_data_id,
suffix_msg=_("失败")
)
notify_manager(message)
raise error_codes.APIError('{}dataid{}:{}'.format(_("申请标准日志采集"), _("失败"), standard_data_id))
# 申请非标准日志采集 dataid
non_standard_data_name = f'{DataType.CLOG.value}_{english_name}'
res2, non_standard_data_id = deploy_plan(username, cc_app_id, non_standard_data_name, DataType.CLOG.value)
if not res2:
message = '''{prefix_msg}dataid{biz}ID:{cc_app_id},{project}:{english_name}{fail},
{reason}:{standard_data_id},{suffix_msg}'''.format(
prefix_msg=_("申请非标准日志采集"),
biz=_("业务"),
cc_app_id=cc_app_id,
project=_("项目名"),
english_name=english_name,
fail=_("失败"),
reason=_("原因"),
standard_data_id=standard_data_id,
suffix_msg=_("请关注")
)
notify_manager(message)
raise error_codes.APIError('{}dataid{}:{}'.format(_("申请非标准日志采集"), _("失败"), standard_data_id))
# 数据平台 V3 API 没有project_id的概念，给一个默认的值
project.data_project_id = 1
project.standard_data_id = standard_data_id
project.standard_data_name = standard_data_name
project.non_standard_data_id = non_standard_data_id
project.non_standard_data_name = non_standard_data_name
project.save()
return True
