def insert_kafka_cluster(deploy_info):
"""
部署kafka集群初始化数据库信息
@param deploy_info: {
"app_id": 业务id,
"app": 业务名称,
"add_type": 0/1,
"cluster_name": 集群名称,
"version": 集群版本,
"target_ips": broker ip列表,
"broker_str": zookeeper 集群信息,
"description": 集群描述,
"bk_username": 创建者,
"task_type":
}
"""
try:
# 绑定事务插入，保证事务的数据一致性
with transaction.atomic():
cluster = KafkaCluster.objects.create(
cluster_name=deploy_info['cluster_name'],
app=deploy_info['app'],
app_id=deploy_info['app_id'],
add_type=deploy_info['add_type'],
zk_list=deploy_info['broker_str'],
broker_cnt=len(deploy_info['target_ips']),
version=deploy_info['version'],
description=deploy_info['description'],
created_by=deploy_info['bk_username'],
)
# 集群信息表插入后 插入broker信息
for broker_ip in deploy_info['target_ips']:
# 在配置平台获取IP的硬件信息
res = get_cc_info_by_ip(deploy_info['bk_username'],
deploy_info['app_id'], broker_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
KafkaBroker.objects.create(
app_id=deploy_info['app_id'],
cluster_name=deploy_info['cluster_name'],
ip=broker_ip,
version=deploy_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
except Exception as err:
logger.error(f"集群数据插入发生异常：{str(err)}")
return None
return cluster.id
def add_es_node(add_info):
"""
扩容后插入新的进程信息
@param add_info:{
"app_id": 业务id,
"app": 业务名称,
"cluster_id": 扩容所在的集群id,
"cluster_name": 扩容所在的集群名称,
"version": 节点版本,
"http_port": es http 集群端口号,
"target_ips": 扩容ip列表,
"role": 扩容节点角色,
"master_str": 集群master节点信息，已逗号隔开,
"bk_username": 任务创建者,
"task_type": 任务类型
}
"""
role = add_info['role']
add_ips = add_info[f"{role}_list"]
try:
with transaction.atomic():
update_data = {f'{role}_cnt': F(f'{role}_cnt') + len(add_ips)}
EsCluster.objects.filter(id=add_info['cluster_id']).update(
**update_data)
for add_node_ip in add_ips:
# 在配置平台获取IP的硬件信息
res = get_cc_info_by_ip(add_info['bk_username'],
add_info['app_id'], add_node_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
EsNodeInfo.objects.create(
app_id=add_info['app_id'],
cluster_name=add_info['cluster_name'],
node_name=f"{add_info['role']}-{add_node_ip}",
role=add_info['role'],
ip=add_node_ip,
version=add_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
return True
except Exception as err:
logger.error(f"ES集群新增节点插入发生异常: {str(err)}")
return False
def check_kafka_add_ip(bk_username, ip_list, app_id, app):
"""
提取公共检测ip的代码来封装，减少重复代码
@param bk_username: ip在配置平台检测需要的用户名称 参数类型： str
@param ip_list: 任务中新加的带检测ip列表，参数类型: list
@param app_id: ip在配置平台检测需要的业务ID 参数类型： int
@param app: ip的配置平台检测需要的业务名称 参数类型： str
"""
for node_ip in ip_list:
if not is_ip(node_ip):
# 存在非法ip
return build_fail_result(f"存在非法IP:{node_ip}")
if KafkaBroker.objects.filter(ip=node_ip).exists():
# ip已录入平台
return build_fail_result(f"平台检测到存在该IP:{node_ip}")
if get_cc_info_by_ip(bk_username=bk_username, app_id=app_id, ip=node_ip)['data']['count'] == 0:
# 节点不属于对应业务，异常退出
return build_fail_result(f"节点不属于对应业务{app}，请自查:{node_ip}")
return None
def add_broker_node(add_info):
"""
扩容后插入新的进程信息
@param add_info:{
"cluster_id": 集群id,
"app_id": 业务id,
"cluster_name": 集群名称,
"version": 版本号,
"target_ips": 扩容ip信list,
"broker_str": zookeeper信息,
"task_type":
"bk_username": 操作者
}
"""
add_cnt = len(add_info['target_ips'])
try:
with transaction.atomic():
KafkaCluster.objects.filter(id=add_info['cluster_id']).update(
broker_cnt=F('broker_cnt') + add_cnt)
for add_broker_ip in add_info['target_ips']:
# 在配置平台获取IP的硬件信息
res = get_cc_info_by_ip(add_info['bk_username'],
add_info['app_id'], add_broker_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
KafkaBroker.objects.create(
app_id=add_info['app_id'],
cluster_name=add_info['cluster_name'],
ip=add_broker_ip,
version=add_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
return True
except Exception as err:
logger.error(f"新增节点插入发生异常: {str(err)}")
return False
def insert_es_cluster(deploy_info):
"""
部署kafka集群初始化数据库信息
@param deploy_info : {
"app_id": 业务id,
"app": 业务名称,
"add_type": 0/1 添加模式,
"cluster_name": 集群名称,
"version": 版本号,
"http_port": es http 集群端口号,
"master_list": master节点列表,
"data_list": data节点列表,
"cold_list": cold节点列表,
"client_list": client节点列表,
"master_str": master节点信息，已逗号隔开,
"account": 集群初始化账号,
"password": 集群初始化账号密码,
"description": 集群描述,
"bk_username": 创建者,
"task_type": 任务类型
}
"""
try:
# 绑定事务插入，保证事务的数据一致性
with transaction.atomic():
cluster = EsCluster.objects.create(
cluster_name=deploy_info['cluster_name'],
app=deploy_info['app'],
app_id=deploy_info['app_id'],
master_list=deploy_info['master_str'],
master_cnt=len(deploy_info['master_list']),
data_cnt=len(deploy_info['data_list']),
cold_cnt=len(deploy_info['cold_list']),
client_cnt=len(deploy_info['client_list']),
version=deploy_info['version'],
http_port=deploy_info['http_port'],
description=deploy_info['description'],
created_by=deploy_info['bk_username'],
)
# 集群信息表插入后，插入node节点信息
for master_ip in deploy_info['master_list']:
res = get_cc_info_by_ip(deploy_info['bk_username'],
deploy_info['app_id'], master_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
EsNodeInfo.objects.create(
app_id=deploy_info['app_id'],
cluster_name=deploy_info['cluster_name'],
node_name=f"master-{master_ip}",
role='master',
ip=master_ip,
version=deploy_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
for data_ip in deploy_info['data_list']:
res = get_cc_info_by_ip(deploy_info['bk_username'],
deploy_info['app_id'], data_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
EsNodeInfo.objects.create(
app_id=deploy_info['app_id'],
cluster_name=deploy_info['cluster_name'],
node_name=f"dn-{data_ip}",
role='data',
ip=data_ip,
version=deploy_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
for cold_ip in deploy_info['cold_list']:
res = get_cc_info_by_ip(deploy_info['bk_username'],
deploy_info['app_id'], cold_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
EsNodeInfo.objects.create(
app_id=deploy_info['app_id'],
cluster_name=deploy_info['cluster_name'],
node_name=f"cold-{cold_ip}",
role='cold',
ip=cold_ip,
version=deploy_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
for client_ip in deploy_info['client_list']:
res = get_cc_info_by_ip(deploy_info['bk_username'],
deploy_info['app_id'], client_ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
EsNodeInfo.objects.create(
app_id=deploy_info['app_id'],
cluster_name=deploy_info['cluster_name'],
node_name=f"client-{client_ip}",
role='client',
ip=client_ip,
version=deploy_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
# 插入集群用户信息
EsRule.objects.create(app=deploy_info['app'],
app_id=deploy_info['app_id'],
cluster_name=deploy_info['cluster_name'],
user_name=deploy_info['account'])
except Exception as err:
logger.error(f"集群数据插入发生异常：{str(err)}")
return None
return cluster.id
def sync_db(node_info, input_info):
"""
获取成功后录入db信息
@param node_info: 执行成功后返回集群json信息 参数类型：str/dict
@param input_info: 执行集群录入任务的任务参数 参数类型：dict
"""
if not isinstance(node_info, dict):
node_info = json.loads(node_info)
master_node_sum = 0
data_node_sum = 0
cold_node_sum = 0
client_node_sum = 0
master_list = []
http_port = input_info['input_http_url'].split(':')[1]
for node in node_info:
node_name = node.get('name')
ip = node.get('ip')
role_list = []
if node.get('node.role') == 'ir' or node.get(
'node.role') == 'r' or node.get('node.role') == '':
role_list.append("client")
client_node_sum += 1
if node.get('node.role').find('c') != -1:
role_list.append("cold")
cold_node_sum += 1
if node.get('node.role').find('d') != -1:
role_list.append("data")
data_node_sum += 1
if node.get('node.role').find('m') != -1:
role_list.append("master")
master_list.append(ip)
master_node_sum += 1
# 尝试获取机器的硬件信息
res = get_cc_info_by_ip(input_info['bk_username'],
input_info['app_id'], ip)
device_class = res['data']['info'][0].get('svr_device_class')
hard_memo = res['data']['info'][0].get('hard_memo')
EsNodeInfo.objects.create(
app_id=int(input_info['app_id']),
cluster_name=input_info['cluster_name'],
node_name=node_name,
role=",".join(role_list),
ip=ip,
version=input_info['version'],
device_class=device_class,
hard_memo=hard_memo,
)
cluster = EsCluster.objects.create(
cluster_name=input_info['cluster_name'],
app=input_info['app'],
app_id=int(input_info['app_id']),
master_list=",".join(master_list),
master_cnt=master_node_sum,
data_cnt=data_node_sum,
cold_cnt=cold_node_sum,
client_cnt=client_node_sum,
version=input_info['version'],
http_port=http_port,
description=input_info['description'],
created_by=input_info['bk_username'],
add_type=0,
)
return cluster.id