def make_deploy_msgs(self,cluster,supertask,deploymsg,status):
#ebs_rolelist=["gangliasrv","nagiossrv","gangliacli","nagioscli"]
task_messages=[]
ebs_rolelist=["keepalived","onecmdb","activemq","redis","nagios","ganglia","mysql","cloudmaster","clmrs","octopus","cmagent","x86master","rabbitmq","cmgather","monitor","AlarmApp","autoAlarm","ceilometer","PerformancePM","notify"]
if status== 1:
ebs_rolelist=["notify","PerformancePM","ceilometer","autoAlarm","AlarmApp","monitor","cmgather","rabbitmq","x86master","cmagent","octopus","clmrs","cloudmaster","mysql","ganglia","nagios","redis","activemq","onecmdb","keepalived"]
#获取当前集群下所有处于已就绪的节点
nodes_to_startorstop=TaskHelper.nodes_to_startorstop(cluster)
#获取所有节点的所有角色集合
nodes_roles=[]
for node in nodes_to_startorstop:
nodes_roles.extend(node.roles)
nodes_roles=list(set(nodes_roles))
logger.info(deploymsg)
for role in nodes_roles:
if role in ebs_rolelist:
task_deployment = supertask.create_subtask(TASK_NAMES.deployment)
db().commit()
newdeploymsg=copy.deepcopy(deploymsg)
newdeploymsg['respond_to']="start_stop_resp"
newdeploymsg['args']['task_uuid']=task_deployment.uuid
deployment_infos=[]
for deployment_info in deploymsg['args']['deployment_info']:
if deployment_info["role"] != role:
newdeploymsg['args']['deployment_info'].remove(deployment_info)
else:
if status == 2:
deployment_info[role]['action']="start"
logger.info(u"匹配到角色{0},节点id {1},开始启动...".format(role,deployment_info["ip"]))
else:
deployment_info[role]['action']="stop"
logger.info(u"匹配到角色{0},节点id {1},开始停止...".format(role,deployment_info["ip"]))
deployment_infos.append(deployment_info)
newdeploymsg['args']['deployment_info']=deployment_infos
task_messages.append(newdeploymsg)
task_deployment = objects.Task.get_by_uid(
task_deployment.id,
fail_if_not_found=True,
lock_for_update=True
)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == TASK_STATUSES.error:
return supertask
task_deployment.cache = newdeploymsg
db().commit()
self.update_cluster_role_status(cluster,role,status)
new_task_messages=[]
logger.info(len(task_messages))
for ebsrole in ebs_rolelist:
for task in task_messages:
if task['args']['deployment_info'][0]['role'] == ebsrole:
new_task_messages.append(task)
return new_task_messages
def execute(self):
#开始执行部署变更
logger.info(u"Trying to start deployment at cluster '{0}'".format(
self.cluster.name or self.cluster.id))
#显示网络信息(openstack部署前执行网络验证)
network_info = self.serialize_network_cfg(self.cluster)
logger.info(u"Network info:\n{0}".format(
jsonutils.dumps(network_info, indent=4)))
self._remove_obsolete_tasks() #obsolete 过时的
supertask = Task(name=TASK_NAMES.deploy, cluster=self.cluster)
db().add(supertask)
nodes_to_delete = TaskHelper.nodes_to_delete(self.cluster)
nodes_to_deploy = TaskHelper.nodes_to_deploy(self.cluster)
nodes_to_provision = TaskHelper.nodes_to_provision(self.cluster)
task_messages = []
#如果是openstack环境,就执行原来流程判断看集群中是否有节点的变化
if self.cluster.cluster_type == 1:
if not any([nodes_to_provision, nodes_to_deploy, nodes_to_delete]):
db().rollback()
raise errors.WrongNodeStatus("No changes to deploy")
# we should have task committed for processing in other threads
db().commit()
TaskHelper.create_action_log(supertask)
# Run validation if user didn't redefine
# provisioning and deployment information
if (not objects.Cluster.get_provisioning_info(self.cluster)
and not objects.Cluster.get_deployment_info(self.cluster)):
try:
if self.cluster.cluster_type == 1:
self.check_before_deployment(supertask)
except errors.CheckBeforeDeploymentError:
db().commit()
return supertask
task_deletion, task_provision, task_deployment = None, None, None
if nodes_to_delete:
objects.TaskCollection.lock_cluster_tasks(self.cluster.id)
# For more accurate progress calculation
task_weight = 0.4
task_deletion = supertask.create_subtask(TASK_NAMES.node_deletion,
weight=task_weight)
logger.debug("Launching deletion task: %s", task_deletion.uuid)
self._call_silently(task_deletion, tasks.DeletionTask)
# we should have task committed for processing in other threads
db().commit()
if nodes_to_provision:
objects.TaskCollection.lock_cluster_tasks(self.cluster.id)
# updating nodes
nodes_to_provision = objects.NodeCollection.lock_nodes(
nodes_to_provision)
objects.NodeCollection.update_slave_nodes_fqdn(nodes_to_provision)
logger.debug("There are nodes to provision: %s",
" ".join([n.fqdn for n in nodes_to_provision]))
# For more accurate progress calulation
task_weight = 0.4
task_provision = supertask.create_subtask(TASK_NAMES.provision,
weight=task_weight)
# we should have task committed for processing in other threads
db().commit()
provision_message = self._call_silently(task_provision,
tasks.ProvisionTask,
nodes_to_provision,
method_name='message')
task_provision = objects.Task.get_by_uid(task_provision.id,
fail_if_not_found=True,
lock_for_update=True)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_provision.status == TASK_STATUSES.error:
return supertask
task_provision.cache = provision_message
db().commit()
task_messages.append(provision_message)
else:
pass
#nodes_to_deploy=self.cluster.nodes
if nodes_to_deploy:
objects.TaskCollection.lock_cluster_tasks(self.cluster.id)
# locking nodes before updating
objects.NodeCollection.lock_nodes(nodes_to_deploy)
# updating nodes
objects.NodeCollection.update_slave_nodes_fqdn(nodes_to_deploy)
logger.debug("There are nodes to deploy: %s",
" ".join([n.fqdn for n in nodes_to_deploy]))
task_deployment = supertask.create_subtask(TASK_NAMES.deployment)
# we should have task committed for processing in other threads
db().commit()
deployment_message = self._call_silently(task_deployment,
tasks.DeploymentTask,
nodes_to_deploy,
method_name='message')
# clusterdeploymsg = ClusterdeployMsg(cluster_id=self.cluster.id,cluster_deploymsg='deployment_message')
# db().add(clusterdeploymsg)
# clusterdeploymsg只要保存状态为new的部署信息,如果第一次部署失败有错误节点
# 第二次点击部署变更的时候会只会发送消息给错误节点,这样的信息覆盖掉第一次
# 完整的部署信息会导致集群启动和停止的失败(因为只发送给错误节点).
logger.info(u'执行部署变更操作,开始操作cluster_deploy_msg表')
data = {}
data['cluster_id'] = self.cluster.id
data['cluster_deploymsg'] = jsonutils.dumps(deployment_message)
clusterdeploymsg = db().query(ClusterdeployMsg).filter_by(
cluster_id=self.cluster.id).first()
if clusterdeploymsg:
if self.cluster.status == CLUSTER_STATUSES.new:
objects.ClusterdeployMsgObject.update(
clusterdeploymsg,
{'cluster_deploymsg': data['cluster_deploymsg']})
else:
objects.ClusterdeployMsgCollection.create(data)
task_deployment = objects.Task.get_by_uid(task_deployment.id,
fail_if_not_found=True,
lock_for_update=True)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == TASK_STATUSES.error:
return supertask
task_deployment.cache = deployment_message
db().commit()
task_messages.append(deployment_message)
else:
pass
if len(nodes_to_delete) <= 0 and len(nodes_to_deploy) <= 0:
#这里不能单纯用nodes_to_deploy是否为空来判断是启动或者停止
#因为nodes_to_delete不为空而nodes_to_deploy为空的话依然会
#执行下面的代码,此种情况程序会出现异常
oprolename = web.cookies().get("oprolename")
opaction = web.cookies().get("opaction")
nodes_to_startorstop = TaskHelper.nodes_to_startorstop(
self.cluster)
if oprolename != "all":
logger.info(u'这里执行的是单角色启动或者停止部署')
task_deployment = supertask.create_subtask(
TASK_NAMES.deployment)
# we should have task committed for processing in other threads
# openstack定制化环境点击部署变更和启动,停止按钮没有执行
# 此程序,而是直接通过调用/opt/start.py执行
# 目前只有cloudmaster和ebs环境会执行此处代码
db().commit()
deployment_message = self._call_silently(task_deployment,
tasks.DeploymentTask,
nodes_to_startorstop,
method_name='message')
deploymsg = deployment_message
deploymsg['respond_to'] = "start_stop_resp"
deploymsg['args']['task_uuid'] = task_deployment.uuid
#deployment_info[]是列表,这个列表中含有的元素都是字典
#角色下面的label参数就只是在此处添加和修改的.
deployment_info_old_list = deploymsg['args']['deployment_info']
deployment_info_list = []
nodeuids = []
for key, item in groupby(deployment_info_old_list,
itemgetter('uid')):
nodeuids.append(key)
deployment_info_list = deployment_info_old_list[0:len(nodeuids
)]
#此处删除和启动停止无关的角色信息
deploymsg['args']['deployment_info'] = []
for i in range(len(deployment_info_list)):
deployment_info = deployment_info_list[i]
deployment_info["role"] = oprolename
deployment_info["uid"] = nodeuids[i]
deploymsg['args']['deployment_info'].append(
deployment_info)
for deployment_info in deployment_info_list: #此处是一个列表
deployment_info_keys = deployment_info.keys()
changelable_keys = []
operationalrole = ""
for key in deployment_info_keys:
if key.lower() == oprolename.lower():
operationalrole = key
else:
changelable_keys.append(key)
deployment_info[operationalrole]['action'] = opaction
deployment_info[operationalrole]['label'] = '0'
for key in changelable_keys:
if type(deployment_info[key]
) == dict and deployment_info[key].get(
'label') != None:
deployment_info[key]['label'] = '1'
logger.info(deployment_info[operationalrole]['action'])
logger.info(oprolename)
task_deployment = objects.Task.get_by_uid(
task_deployment.id,
fail_if_not_found=True,
lock_for_update=True)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == TASK_STATUSES.error:
return supertask
task_deployment.cache = deploymsg
db().commit()
task_messages.append(deploymsg)
else:
logger.info(u'这里执行的是一键启动和停止操作')
serialized_cluster = deployment_serializers.serialize(
self.cluster, nodes_to_startorstop)
pre_deployment = plugins_serializers.pre_deployment_serialize(
self.cluster, nodes_to_startorstop)
post_deployment = plugins_serializers.post_deployment_serialize(
self.cluster, nodes_to_startorstop)
deployment_message = self.make_astute_message(
'deploy', 'deploy_resp', {
'deployment_info': serialized_cluster,
'pre_deployment': pre_deployment,
'post_deployment': post_deployment
})
if self.cluster.cluster_type == 3:
ebs_start = ebs.StartAllRole()
if opaction == "stop":
task_messages = ebs_start.make_deploy_msgs(
self.cluster, supertask, deployment_message, 1)
else:
task_messages = ebs_start.make_deploy_msgs(
self.cluster, supertask, deployment_message, 2)
else:
pass
if nodes_to_provision:
nodes_to_provision = objects.NodeCollection.lock_nodes(
nodes_to_provision)
for node in nodes_to_provision:
node.status = NODE_STATUSES.provisioning
db().commit()
objects.Cluster.get_by_uid(self.cluster.id,
fail_if_not_found=True,
lock_for_update=True)
self.cluster.status = CLUSTER_STATUSES.deployment
db().add(self.cluster)
db().commit()
if task_messages:
rpc.cast('naily', task_messages)
logger.debug(u"Deployment: task to deploy cluster '{0}' is {1}".format(
self.cluster.name or self.cluster.id, supertask.uuid))
return supertask
def execute(self):
# 开始执行部署变更
logger.info(u"Trying to start deployment at cluster '{0}'".format(self.cluster.name or self.cluster.id))
# 显示网络信息(openstack部署前执行网络验证)
network_info = self.serialize_network_cfg(self.cluster)
logger.info(u"Network info:\n{0}".format(jsonutils.dumps(network_info, indent=4)))
self._remove_obsolete_tasks() # obsolete 过时的
supertask = Task(name=TASK_NAMES.deploy, cluster=self.cluster)
db().add(supertask)
nodes_to_delete = TaskHelper.nodes_to_delete(self.cluster)
nodes_to_deploy = TaskHelper.nodes_to_deploy(self.cluster)
nodes_to_provision = TaskHelper.nodes_to_provision(self.cluster)
task_messages = []
# 如果是openstack环境,就执行原来流程判断看集群中是否有节点的变化
if self.cluster.cluster_type == 1:
if not any([nodes_to_provision, nodes_to_deploy, nodes_to_delete]):
db().rollback()
raise errors.WrongNodeStatus("No changes to deploy")
# we should have task committed for processing in other threads
db().commit()
TaskHelper.create_action_log(supertask)
# Run validation if user didn't redefine
# provisioning and deployment information
if not objects.Cluster.get_provisioning_info(self.cluster) and not objects.Cluster.get_deployment_info(
self.cluster
):
try:
if self.cluster.cluster_type == 1:
self.check_before_deployment(supertask)
except errors.CheckBeforeDeploymentError:
db().commit()
return supertask
task_deletion, task_provision, task_deployment = None, None, None
if nodes_to_delete:
objects.TaskCollection.lock_cluster_tasks(self.cluster.id)
# For more accurate progress calculation
task_weight = 0.4
task_deletion = supertask.create_subtask(TASK_NAMES.node_deletion, weight=task_weight)
logger.debug("Launching deletion task: %s", task_deletion.uuid)
self._call_silently(task_deletion, tasks.DeletionTask)
# we should have task committed for processing in other threads
db().commit()
if nodes_to_provision:
objects.TaskCollection.lock_cluster_tasks(self.cluster.id)
# updating nodes
nodes_to_provision = objects.NodeCollection.lock_nodes(nodes_to_provision)
objects.NodeCollection.update_slave_nodes_fqdn(nodes_to_provision)
logger.debug("There are nodes to provision: %s", " ".join([n.fqdn for n in nodes_to_provision]))
# For more accurate progress calulation
task_weight = 0.4
task_provision = supertask.create_subtask(TASK_NAMES.provision, weight=task_weight)
# we should have task committed for processing in other threads
db().commit()
provision_message = self._call_silently(
task_provision, tasks.ProvisionTask, nodes_to_provision, method_name="message"
)
task_provision = objects.Task.get_by_uid(task_provision.id, fail_if_not_found=True, lock_for_update=True)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_provision.status == TASK_STATUSES.error:
return supertask
task_provision.cache = provision_message
db().commit()
task_messages.append(provision_message)
else:
pass
# nodes_to_deploy=self.cluster.nodes
if nodes_to_deploy:
objects.TaskCollection.lock_cluster_tasks(self.cluster.id)
# locking nodes before updating
objects.NodeCollection.lock_nodes(nodes_to_deploy)
# updating nodes
objects.NodeCollection.update_slave_nodes_fqdn(nodes_to_deploy)
logger.debug("There are nodes to deploy: %s", " ".join([n.fqdn for n in nodes_to_deploy]))
task_deployment = supertask.create_subtask(TASK_NAMES.deployment)
# we should have task committed for processing in other threads
db().commit()
deployment_message = self._call_silently(
task_deployment, tasks.DeploymentTask, nodes_to_deploy, method_name="message"
)
# clusterdeploymsg = ClusterdeployMsg(cluster_id=self.cluster.id,cluster_deploymsg='deployment_message')
# db().add(clusterdeploymsg)
# clusterdeploymsg只要保存状态为new的部署信息,如果第一次部署失败有错误节点
# 第二次点击部署变更的时候会只会发送消息给错误节点,这样的信息覆盖掉第一次
# 完整的部署信息会导致集群启动和停止的失败(因为只发送给错误节点).
logger.info(u"执行部署变更操作,开始操作cluster_deploy_msg表")
data = {}
data["cluster_id"] = self.cluster.id
data["cluster_deploymsg"] = jsonutils.dumps(deployment_message)
clusterdeploymsg = db().query(ClusterdeployMsg).filter_by(cluster_id=self.cluster.id).first()
if clusterdeploymsg:
if self.cluster.status == CLUSTER_STATUSES.new:
objects.ClusterdeployMsgObject.update(
clusterdeploymsg, {"cluster_deploymsg": data["cluster_deploymsg"]}
)
else:
objects.ClusterdeployMsgCollection.create(data)
task_deployment = objects.Task.get_by_uid(task_deployment.id, fail_if_not_found=True, lock_for_update=True)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == TASK_STATUSES.error:
return supertask
task_deployment.cache = deployment_message
db().commit()
task_messages.append(deployment_message)
else:
pass
if len(nodes_to_delete) <= 0 and len(nodes_to_deploy) <= 0:
# 这里不能单纯用nodes_to_deploy是否为空来判断是启动或者停止
# 因为nodes_to_delete不为空而nodes_to_deploy为空的话依然会
# 执行下面的代码,此种情况程序会出现异常
oprolename = web.cookies().get("oprolename")
opaction = web.cookies().get("opaction")
nodes_to_startorstop = TaskHelper.nodes_to_startorstop(self.cluster)
if oprolename != "all":
logger.info(u"这里执行的是单角色启动或者停止部署")
task_deployment = supertask.create_subtask(TASK_NAMES.deployment)
# we should have task committed for processing in other threads
# openstack定制化环境点击部署变更和启动,停止按钮没有执行
# 此程序,而是直接通过调用/opt/start.py执行
# 目前只有cloudmaster和ebs环境会执行此处代码
db().commit()
deployment_message = self._call_silently(
task_deployment, tasks.DeploymentTask, nodes_to_startorstop, method_name="message"
)
deploymsg = deployment_message
deploymsg["respond_to"] = "start_stop_resp"
deploymsg["args"]["task_uuid"] = task_deployment.uuid
# deployment_info[]是列表,这个列表中含有的元素都是字典
# 角色下面的label参数就只是在此处添加和修改的.
deployment_info_old_list = deploymsg["args"]["deployment_info"]
deployment_info_list = []
nodeuids = []
for key, item in groupby(deployment_info_old_list, itemgetter("uid")):
nodeuids.append(key)
deployment_info_list = deployment_info_old_list[0 : len(nodeuids)]
# 此处删除和启动停止无关的角色信息
deploymsg["args"]["deployment_info"] = []
for i in range(len(deployment_info_list)):
deployment_info = deployment_info_list[i]
deployment_info["role"] = oprolename
deployment_info["uid"] = nodeuids[i]
deploymsg["args"]["deployment_info"].append(deployment_info)
for deployment_info in deployment_info_list: # 此处是一个列表
deployment_info_keys = deployment_info.keys()
changelable_keys = []
operationalrole = ""
for key in deployment_info_keys:
if key.lower() == oprolename.lower():
operationalrole = key
else:
changelable_keys.append(key)
deployment_info[operationalrole]["action"] = opaction
deployment_info[operationalrole]["label"] = "0"
for key in changelable_keys:
if type(deployment_info[key]) == dict and deployment_info[key].get("label") != None:
deployment_info[key]["label"] = "1"
logger.info(deployment_info[operationalrole]["action"])
logger.info(oprolename)
task_deployment = objects.Task.get_by_uid(
task_deployment.id, fail_if_not_found=True, lock_for_update=True
)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == TASK_STATUSES.error:
return supertask
task_deployment.cache = deploymsg
db().commit()
task_messages.append(deploymsg)
else:
logger.info(u"这里执行的是一键启动和停止操作")
serialized_cluster = deployment_serializers.serialize(self.cluster, nodes_to_startorstop)
pre_deployment = plugins_serializers.pre_deployment_serialize(self.cluster, nodes_to_startorstop)
post_deployment = plugins_serializers.post_deployment_serialize(self.cluster, nodes_to_startorstop)
deployment_message = self.make_astute_message(
"deploy",
"deploy_resp",
{
"deployment_info": serialized_cluster,
"pre_deployment": pre_deployment,
"post_deployment": post_deployment,
},
)
if self.cluster.cluster_type == 3:
ebs_start = ebs.StartAllRole()
if opaction == "stop":
task_messages = ebs_start.make_deploy_msgs(self.cluster, supertask, deployment_message, 1)
else:
task_messages = ebs_start.make_deploy_msgs(self.cluster, supertask, deployment_message, 2)
elif self.cluster.cluster_type == 2:
cloud_start = cld.StartAllRole()
if opaction == "stop":
task_messages = cloud_start.make_deploy_msgs(self.cluster, supertask, deployment_message, 1)
else:
task_messages = cloud_start.make_deploy_msgs(self.cluster, supertask, deployment_message, 2)
else:
pass
if nodes_to_provision:
nodes_to_provision = objects.NodeCollection.lock_nodes(nodes_to_provision)
for node in nodes_to_provision:
node.status = NODE_STATUSES.provisioning
db().commit()
objects.Cluster.get_by_uid(self.cluster.id, fail_if_not_found=True, lock_for_update=True)
self.cluster.status = CLUSTER_STATUSES.deployment
db().add(self.cluster)
db().commit()
if task_messages:
rpc.cast("naily", task_messages)
logger.debug(
u"Deployment: task to deploy cluster '{0}' is {1}".format(
self.cluster.name or self.cluster.id, supertask.uuid
)
)
return supertask
def make_deploy_msgs(self, cluster, supertask, deploymsg, status):
#ebs_rolelist=["gangliasrv","nagiossrv","gangliacli","nagioscli"]
task_messages = []
ebs_rolelist = [
"gangliasrv", "nagiossrv", "gangliacli", "nagioscli", "management",
"watcher", "tgtd", "rsyslogsrv", "rsyslogcli"
]
if status == 1:
ebs_rolelist = [
"rsyslogcli", "rsyslogsrv", "tgtd", "watcher", "management",
"nagioscli", "gangliacli", "nagiossrv", "gangliasrv"
]
#获取当前集群下所有处于已就绪的节点
nodes_to_startorstop = TaskHelper.nodes_to_startorstop(cluster)
#获取所有节点的所有角色集合
nodes_roles = []
for node in nodes_to_startorstop:
nodes_roles.extend(node.roles)
nodes_roles = list(set(nodes_roles))
logger.info(deploymsg)
for role in nodes_roles:
if role in ebs_rolelist:
task_deployment = supertask.create_subtask(
TASK_NAMES.deployment)
db().commit()
newdeploymsg = copy.deepcopy(deploymsg)
newdeploymsg['respond_to'] = "start_stop_resp"
newdeploymsg['args']['task_uuid'] = task_deployment.uuid
deployment_infos = []
for deployment_info in deploymsg['args']['deployment_info']:
if deployment_info["role"] != role:
newdeploymsg['args']['deployment_info'].remove(
deployment_info)
else:
if status == 2:
deployment_info[role]['action'] = "start"
logger.info(u"匹配到角色{0},节点id {1},开始启动...".format(
role, deployment_info["ip"]))
else:
deployment_info[role]['action'] = "stop"
logger.info(u"匹配到角色{0},节点id {1},开始停止...".format(
role, deployment_info["ip"]))
deployment_infos.append(deployment_info)
newdeploymsg['args']['deployment_info'] = deployment_infos
task_messages.append(newdeploymsg)
task_deployment = objects.Task.get_by_uid(
task_deployment.id,
fail_if_not_found=True,
lock_for_update=True)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == TASK_STATUSES.error:
return supertask
task_deployment.cache = newdeploymsg
db().commit()
self.update_cluster_role_status(cluster, role, status)
new_task_messages = []
logger.info(len(task_messages))
for ebsrole in ebs_rolelist:
for task in task_messages:
if task['args']['deployment_info'][0]['role'] == ebsrole:
new_task_messages.append(task)
return new_task_messages
