def deployment_tasks(self, plugins, stage):
tasks = []
for plugin in plugins:
plugin_tasks = filter(lambda t: (t['stage'] == stage),
plugin.tasks)
for task in plugin_tasks:
uids = get_uids_for_roles(self.nodes, task['role'])
if not uids:
continue
if task['type'] == 'shell':
tasks.append(
self.serialize_task(
plugin, task,
templates.make_shell_task(
uids, task, plugin.slaves_scripts_path)))
elif task['type'] == 'puppet':
tasks.append(
self.serialize_task(
plugin, task,
templates.make_puppet_task(
uids, task, plugin.slaves_scripts_path)))
else:
logger.warn('Task is skipped {0}, because its type is '
'not supported').format(task)
return tasks
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 202 (environment reset initiated)
* 400 (can't reset environment)
* 404 (environment not found in db)
"""
cluster = self.get_object_or_404(Cluster, cluster_id)
try:
logger.info(
u"Trying to reset environment '{0}'".format(
cluster_id
)
)
task_manager = ResetEnvironmentTaskManager(
cluster_id=cluster.id
)
task = task_manager.execute()
except Exception as exc:
logger.warn(u'Error during execution '
u'environment resetting '
u'task: {0}'.format(str(exc)))
raise web.badrequest(str(exc))
raise web.webapi.HTTPError(
status="202 Accepted",
data=TaskHandler.render(task)
)
def get_node_volumes(node):
"""Helper for retrieving node volumes in correct order.
If spaces don't defained for role, will be used
partitioning for role `other`.
"""
node_volumes = []
role_volumes = node.cluster.release.volumes_metadata[
'volumes_roles_mapping']
roles = node.roles + node.pending_roles
for role in roles:
if not role_volumes.get(role):
continue
for volume in role_volumes[role]:
if volume not in node_volumes:
node_volumes.append(volume)
# Use role other
if not node_volumes:
logger.warn('Cannot find volumes for node: %s assigning default '
'volumes' % (node.full_name))
for volume in role_volumes['other']:
node_volumes.append(volume)
return node_volumes
def _get_vlan_splinters_desc(cls, use_vlan_splinters, iface,
cluster):
iface_attrs = {}
if use_vlan_splinters in ('disabled', 'kernel_lt'):
iface_attrs['vlan_splinters'] = 'off'
return iface_attrs
iface_attrs['vlan_splinters'] = 'auto'
trunks = [0]
if use_vlan_splinters == 'hard':
for ng in iface.assigned_networks_list:
if ng.name == 'private':
vlan_range = cluster.network_config.vlan_range
trunks.extend(xrange(*vlan_range))
trunks.append(vlan_range[1])
else:
if ng.vlan_start in (0, None):
continue
trunks.append(ng.vlan_start)
elif use_vlan_splinters == 'soft':
pass
else:
logger.warn('Invalid vlan_splinters value: %s', use_vlan_splinters)
return {}
iface_attrs['trunks'] = trunks
return iface_attrs
def deployment_tasks(self, plugins, stage):
plugin_tasks = []
sorted_plugins = sorted(plugins, key=lambda p: p.plugin.name)
for plugin in sorted_plugins:
stage_tasks = filter(lambda t: t['stage'].startswith(stage),
plugin.tasks)
plugin_tasks.extend(self._set_tasks_defaults(plugin, stage_tasks))
sorted_tasks = self._sort_by_stage_postfix(plugin_tasks)
for task in sorted_tasks:
make_task = None
uids = self.resolver.resolve(task.get('tags', task['role']))
if not uids:
continue
if task['type'] == 'shell':
make_task = templates.make_shell_task
elif task['type'] == 'puppet':
make_task = templates.make_puppet_task
elif task['type'] == 'reboot':
make_task = templates.make_reboot_task
else:
logger.warn(
'Task is skipped %s, because its type is '
'not supported', task)
if make_task:
yield self._serialize_task(make_task(list(uids), task), task)
def _get_node_uids_for_plugin_tasks(self, plugin):
# TODO(aroma): remove concatenation of tasks when unified way of
# processing will be introduced for deployment tasks and existing
# plugin tasks
tasks_to_process = plugin.tasks + plugin.deployment_tasks
roles = []
for task in tasks_to_process:
# plugin tasks may store information about node
# role not only in `role` key but also in `groups`
task_role = task.get('role', task.get('groups'))
if task_role == consts.TASK_ROLES.all:
# just return all nodes
return self.role_resolver.resolve(consts.TASK_ROLES.all)
elif task_role == consts.TASK_ROLES.master:
# NOTE(aroma): pre-deployment tasks should not be executed on
# master node because in some cases it leads to errors due to
# commands need to be run are not compatible with master node
# OS (CentOS). E.g. of such situation - create repository
# executes `apt-get update` which fails on CentOS
continue
elif isinstance(task_role, list):
roles.extend(task_role)
# if task has 'skipped' status it is allowed that 'roles' and
# 'groups' are not be specified
elif task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped:
logger.warn(
'Wrong roles format in task %s: either '
'`roles` or `groups` must be specified and contain '
'a list of roles or "*"',
task)
return self.role_resolver.resolve(roles)
def _get_node_uids_for_plugin_tasks(self, plugin):
# TODO(aroma): remove concatenation of tasks when unified way of
# processing will be introduced for deployment tasks and existing
# plugin tasks
tasks_to_process = plugin.tasks + plugin.deployment_tasks
roles = []
for task in tasks_to_process:
# plugin tasks may store information about node
# role not only in `role` key but also in `groups`
task_role = task.get('role', task.get('groups'))
if task_role == consts.TASK_ROLES.all:
# just return all nodes
return self.role_resolver.resolve(consts.TASK_ROLES.all)
elif task_role == consts.TASK_ROLES.master:
# NOTE(aroma): pre-deployment tasks should not be executed on
# master node because in some cases it leads to errors due to
# commands need to be run are not compatible with master node
# OS (CentOS). E.g. of such situation - create repository
# executes `apt-get update` which fails on CentOS
continue
elif isinstance(task_role, list):
roles.extend(task_role)
# if task has 'skipped' status it is allowed that 'roles' and
# 'groups' are not be specified
elif task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped:
logger.warn(
'Wrong roles format in task %s: either '
'`roles` or `groups` must be specified and contain '
'a list of roles or "*"', task)
return self.role_resolver.resolve(roles)
def deployment_tasks(self, plugins, stage):
tasks = []
plugin_tasks = []
sorted_plugins = sorted(plugins, key=lambda p: p.plugin.name)
for plugin in sorted_plugins:
stage_tasks = filter(
lambda t: t['stage'].startswith(stage), plugin.tasks)
plugin_tasks.extend(self._set_tasks_defaults(plugin, stage_tasks))
sorted_tasks = self._sort_by_stage_postfix(plugin_tasks)
for task in sorted_tasks:
make_task = None
uids = get_uids_for_roles(self.nodes, task['role'])
if not uids:
continue
if task['type'] == 'shell':
make_task = templates.make_shell_task
elif task['type'] == 'puppet':
make_task = templates.make_puppet_task
elif task['type'] == 'reboot':
make_task = templates.make_reboot_task
else:
logger.warn('Task is skipped {0}, because its type is '
'not supported').format(task)
if make_task:
tasks.append(self._serialize_task(make_task(uids, task), task))
return tasks
def PUT(self, cluster_id):
cluster = self.get_object_or_404(
Cluster,
cluster_id,
log_404=(
"warning",
"Error: there is no cluster "
"with id '{0}' in DB.".format(cluster_id)))
try:
network_info = \
NetworkConfigurationSerializer.serialize_for_cluster(
cluster
)
logger.info(
u"Network info:\n{0}".format(
json.dumps(network_info, indent=4)
)
)
task_manager = DeploymentTaskManager(
cluster_id=cluster.id
)
task = task_manager.execute()
except Exception as exc:
logger.warn(u'ClusterChangesHandler: error while execution'
' deploy task: {0}'.format(str(exc)))
raise web.badrequest(str(exc))
return TaskHandler.render(task)
def get_node_volumes(node):
"""Helper for retrieving node volumes in correct order.
If spaces don't defained for role, will be used
partitioning for role `other`.
"""
node_volumes = []
role_volumes = node.cluster.release.volumes_metadata[
'volumes_roles_mapping']
roles = node.roles + node.pending_roles
for role in roles:
if not role_volumes.get(role):
continue
for volume in role_volumes[role]:
if volume not in node_volumes:
node_volumes.append(volume)
# Use role other
if not node_volumes:
logger.warn('Cannot find volumes for node: %s assigning default '
'volumes' % (node.full_name))
for volume in role_volumes['other']:
node_volumes.append(volume)
return node_volumes
def provision_resp(cls, **kwargs):
logger.info(
"RPC method provision_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
task = get_task_by_uuid(task_uuid)
for node in nodes:
uid = node.get('uid')
node_db = db().query(Node).get(uid)
if not node_db:
logger.warn('Task with uid "{0}" not found'.format(uid))
continue
if node.get('status') == 'error':
node_db.status = 'error'
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
TaskHelper.update_task_status(task.uuid, status, progress, message)
def get_node_spaces(node):
"""Helper for retrieving node volumes.
If spaces don't defained for role, will be used
partitioning for role `other`.
Sets key `_allocate_size` which used only for internal calculation
and not used in partitioning system.
"""
node_spaces = []
role_mapping = node.cluster.release.volumes_metadata[
'volumes_roles_mapping']
all_spaces = node.cluster.release.volumes_metadata['volumes']
for role in node.all_roles:
if not role_mapping.get(role):
continue
for volume in role_mapping[role]:
space = find_space_by_id(all_spaces, volume['id'])
if space not in node_spaces:
space['_allocate_size'] = get_allocate_size(node, volume)
node_spaces.append(space)
# Use role `other`
if not node_spaces:
logger.warn('Cannot find volumes for node: %s assigning default '
'volumes' % (node.full_name))
for volume in role_mapping['other']:
space = find_space_by_id(all_spaces, volume['id'])
space['_allocate_size'] = get_allocate_size(node, volume)
node_spaces.append(space)
return node_spaces
def check_range_in_use_already(cidr_range):
for n in used_nets:
if cls.is_range_intersection(n, cidr_range):
logger.warn(
"IP range {0} is in use already".format(cidr_range))
break
used_nets.append(cidr_range)
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 200 (task successfully executed)
* 404 (cluster not found in db)
* 400 (failed to execute task)
"""
cluster = self.get_object_or_404(
Cluster,
cluster_id,
log_404=(
"warning",
"Error: there is no cluster "
"with id '{0}' in DB.".format(cluster_id)))
try:
network_info = \
NetworkConfigurationSerializer.serialize_for_cluster(
cluster
)
logger.info(
u"Network info:\n{0}".format(
json.dumps(network_info, indent=4)
)
)
task_manager = DeploymentTaskManager(
cluster_id=cluster.id
)
task = task_manager.execute()
except Exception as exc:
logger.warn(u'ClusterChangesHandler: error while execution'
' deploy task: {0}'.format(str(exc)))
raise web.badrequest(str(exc))
return TaskHandler.render(task)
def _get_vlan_splinters_desc(cls, use_vlan_splinters, iface, cluster):
iface_attrs = {}
if use_vlan_splinters in ('disabled', 'kernel_lt'):
iface_attrs['vlan_splinters'] = 'off'
return iface_attrs
iface_attrs['vlan_splinters'] = 'auto'
trunks = [0]
if use_vlan_splinters == 'hard':
for ng in iface.assigned_networks_list:
if ng.name == 'private':
vlan_range = cluster.network_config.vlan_range
trunks.extend(xrange(*vlan_range))
trunks.append(vlan_range[1])
else:
if ng.vlan_start in (0, None):
continue
trunks.append(ng.vlan_start)
elif use_vlan_splinters == 'soft':
pass
else:
logger.warn('Invalid vlan_splinters value: %s', use_vlan_splinters)
return {}
iface_attrs['trunks'] = trunks
return iface_attrs
def get_node_spaces(node):
"""Helper for retrieving node volumes.
If spaces don't defained for role, will be used
partitioning for role `other`.
Sets key `_allocate_size` which used only for internal calculation
and not used in partitioning system.
"""
node_spaces = []
role_mapping = node.cluster.release.volumes_metadata[
'volumes_roles_mapping']
all_spaces = node.cluster.release.volumes_metadata['volumes']
for role in node.all_roles:
if not role_mapping.get(role):
continue
for volume in role_mapping[role]:
space = find_space_by_id(all_spaces, volume['id'])
if space not in node_spaces:
space['_allocate_size'] = get_allocate_size(node, volume)
node_spaces.append(space)
# Use role `other`
if not node_spaces:
logger.warn('Cannot find volumes for node: %s assigning default '
'volumes' % (node.full_name))
for volume in role_mapping['other']:
space = find_space_by_id(all_spaces, volume['id'])
space['_allocate_size'] = get_allocate_size(node, volume)
node_spaces.append(space)
return node_spaces
def get_uids_for_tasks(nodes, tasks):
"""Return node uids where particular tasks should be executed
:param nodes: list of Node db objects
:param tasks: list of dicts
:returns: list of strings
"""
roles = []
for task in tasks:
# plugin tasks may store information about node
# role not only in `role` key but also in `groups`
task_role = task.get('role', task.get('groups'))
if task_role == consts.ALL_ROLES:
return get_uids_for_roles(nodes, consts.ALL_ROLES)
elif task_role == consts.MASTER_ROLE:
return [consts.MASTER_ROLE]
elif isinstance(task_role, list):
roles.extend(task_role)
# if task has 'skipped' status it is allowed that 'roles' and
# 'groups' are not be specified
elif task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped:
logger.warn(
'Wrong roles format in task %s: either '
'`roles` or `groups` must be specified and contain '
'a list of roles or "*"',
task)
return get_uids_for_roles(nodes, roles)
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 200 (task successfully executed)
* 404 (cluster not found in db)
* 400 (failed to execute task)
"""
cluster = self.get_object_or_404(
Cluster,
cluster_id,
log_404=("warning", "Error: there is no cluster " "with id '{0}' in DB.".format(cluster_id)),
)
if cluster.net_provider == "nova_network":
net_serializer = NovaNetworkConfigurationSerializer
elif cluster.net_provider == "neutron":
net_serializer = NeutronNetworkConfigurationSerializer
try:
network_info = net_serializer.serialize_for_cluster(cluster)
logger.info(u"Network info:\n{0}".format(json.dumps(network_info, indent=4)))
task_manager = ApplyChangesTaskManager(cluster_id=cluster.id)
task = task_manager.execute()
except Exception as exc:
logger.warn(u"ClusterChangesHandler: error while execution" " deploy task: {0}".format(str(exc)))
raise web.badrequest(str(exc))
return TaskHandler.render(task)
def deployment_tasks(self, plugins, stage):
plugin_tasks = []
sorted_plugins = sorted(plugins, key=lambda p: p.plugin.name)
for plugin in sorted_plugins:
stage_tasks = filter(
lambda t: t['stage'].startswith(stage), plugin.tasks)
plugin_tasks.extend(self._set_tasks_defaults(plugin, stage_tasks))
sorted_tasks = self._sort_by_stage_postfix(plugin_tasks)
for task in sorted_tasks:
make_task = None
uids = self.role_resolver.resolve(task['role'])
if not uids:
continue
if task['type'] == 'shell':
make_task = templates.make_shell_task
elif task['type'] == 'puppet':
make_task = templates.make_puppet_task
elif task['type'] == 'reboot':
make_task = templates.make_reboot_task
else:
logger.warn('Task is skipped %s, because its type is '
'not supported', task)
if make_task:
yield self._serialize_task(make_task(uids, task), task)
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 202 (deployment stopping initiated)
* 400 (can't stop deployment)
* 404 (environment not found in db)
"""
cluster = self.get_object_or_404(Cluster, cluster_id)
try:
logger.info(
u"Trying to stop deployment "
u"on environment '{0}'".format(
cluster_id
)
)
task_manager = StopDeploymentTaskManager(
cluster_id=cluster.id
)
task = task_manager.execute()
except errors.StopAlreadyRunning as exc:
err = web.conflict
err.message = exc.message
raise err
except Exception as exc:
logger.warn(u'Error during execution '
u'deployment stopping task: {0}'.format(str(exc)))
raise web.badrequest(str(exc))
raise web.webapi.HTTPError(
status="202 Accepted",
data=TaskHandler.render(task)
)
def PUT(self):
"""Executes update tasks for specified resources.
:http: * 200 (OK)
* 202 (Accepted)
* 400 (Invalid data)
* 404 (Object dependencies not found)
"""
graph_type = web.input(graph_type=None).graph_type or None
filters = self.checked_data(self.validator.validate_execute)
cluster = self.get_object_or_404(
objects.Cluster, filters['cluster_id'])
# Execute upload task for nodes
task_manager = self.task_manager(cluster_id=cluster.id)
try:
task = task_manager.execute(filters, graph_type=graph_type)
except Exception as exc:
logger.warn(
u'Cannot execute %s task nodes: %s',
self.task_manager.__name__, traceback.format_exc())
raise self.http(400, six.text_type(exc))
self.raise_task(task)
def get_uids_for_tasks(nodes, tasks):
"""Return node uids where particular tasks should be executed
:param nodes: list of Node db objects
:param tasks: list of dicts
:returns: list of strings
"""
roles = []
for task in tasks:
# plugin tasks may store information about node
# role not only in `role` key but also in `groups`
task_role = task.get('role', task.get('groups'))
if task_role == consts.ALL_ROLES:
return get_uids_for_roles(nodes, consts.ALL_ROLES)
elif task_role == consts.MASTER_ROLE:
return [consts.MASTER_ROLE]
elif isinstance(task_role, list):
roles.extend(task_role)
# if task has 'skipped' status it is allowed that 'roles' and
# 'groups' are not be specified
elif task['type'] != consts.ORCHESTRATOR_TASK_TYPES.skipped:
logger.warn(
'Wrong roles format in task %s: either '
'`roles` or `groups` must be specified and contain '
'a list of roles or "*"',
task)
return get_uids_for_roles(nodes, roles)
def _get_vlan_splinters_desc(cls, use_vlan_splinters, iface, cluster):
iface_attrs = {}
if use_vlan_splinters in ("disabled", "kernel_lt"):
iface_attrs["vlan_splinters"] = "off"
return iface_attrs
iface_attrs["vlan_splinters"] = "auto"
trunks = [0]
if use_vlan_splinters == "hard":
for ng in iface.assigned_networks_list:
if (
ng.name == "private"
and cluster.network_config.segmentation_type == consts.NEUTRON_SEGMENT_TYPES.vlan
):
vlan_range = cluster.network_config.vlan_range
trunks.extend(xrange(*vlan_range))
trunks.append(vlan_range[1])
else:
if ng.vlan_start in (0, None):
continue
trunks.append(ng.vlan_start)
elif use_vlan_splinters == "soft":
pass
else:
logger.warn("Invalid vlan_splinters value: %s", use_vlan_splinters)
return {}
iface_attrs["trunks"] = trunks
return iface_attrs
def check_range_in_use_already(cidr_range):
for n in used_nets:
if cls.is_range_intersection(n, cidr_range):
logger.warn("IP range {0} is in use already".format(
cidr_range))
break
used_nets.append(cidr_range)
def provision_resp(cls, **kwargs):
logger.info("RPC method provision_resp received: %s" %
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
task = objects.Task.get_by_uuid(task_uuid,
fail_if_not_found=True,
lock_for_update=True)
# if task was failed on master node then we should
# mark all cluster's nodes in error state
master = next((n for n in nodes if n['uid'] == consts.MASTER_ROLE), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code
if master:
nodes.remove(master)
if master.get('status') == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
progress = 100
# lock nodes for updating
q_nodes = objects.NodeCollection.filter_by_id_list(
None, [n['uid'] for n in nodes])
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
for node in nodes:
uid = node.get('uid')
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warn('Node with uid "{0}" not found'.format(uid))
continue
if node.get('status') == consts.TASK_STATUSES.error:
node_db.status = consts.TASK_STATUSES.error
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_provisioning_task_progress(task)
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def remove_nodes_resp(cls, **kwargs):
logger.info("RPC method remove_nodes_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
error_nodes = kwargs.get('error_nodes') or []
inaccessible_nodes = kwargs.get('inaccessible_nodes') or []
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
for node in nodes:
node_db = db().query(Node).get(node['uid'])
if not node_db:
logger.error(u"Failed to delete node '%s': node doesn't exist",
str(node))
break
db().delete(node_db)
for node in inaccessible_nodes:
# Nodes which not answered by rpc just removed from db
node_db = db().query(Node).get(node['uid'])
if node_db:
logger.warn(u'Node %s not answered by RPC, removing from db',
node_db.human_readable_name)
db().delete(node_db)
for node in error_nodes:
node_db = db().query(Node).get(node['uid'])
if not node_db:
logger.error(
u"Failed to delete node '%s' marked as error from Naily:"
" node doesn't exist", str(node))
break
node_db.pending_deletion = False
node_db.status = 'error'
db().add(node_db)
node['name'] = node_db.name
db().commit()
success_msg = u"No nodes were removed"
err_msg = u"No errors occurred"
if nodes:
success_msg = u"Successfully removed {0} node(s)".format(
len(nodes))
notifier.notify("done", success_msg)
if error_nodes:
err_msg = u"Failed to remove {0} node(s): {1}".format(
len(error_nodes), ', '.join([
n.get('name') or "ID: {0}".format(n['uid'])
for n in error_nodes
]))
notifier.notify("error", err_msg)
if not error_msg:
error_msg = ". ".join([success_msg, err_msg])
TaskHelper.update_task_status(task_uuid, status, progress, error_msg)
def provision_resp(cls, **kwargs):
logger.info(
"RPC method provision_resp received: %s" % jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
task = objects.Task.get_by_uuid(
task_uuid, fail_if_not_found=True, lock_for_update=True)
# if task was failed on master node then we should
# mark all cluster's nodes in error state
master = next((n for n in nodes if n['uid'] == consts.MASTER_ROLE), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code
if master:
nodes.remove(master)
if master.get('status') == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
progress = 100
# lock nodes for updating
q_nodes = objects.NodeCollection.filter_by_id_list(
None, [n['uid'] for n in nodes])
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
for node in nodes:
uid = node.get('uid')
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warn('Node with uid "{0}" not found'.format(uid))
continue
if node.get('status') == consts.TASK_STATUSES.error:
node_db.status = consts.TASK_STATUSES.error
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_provisioning_task_progress(task)
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def _get_task_stage_and_priority(task):
stage_list = task['stage'].split('/')
stage = stage_list[0]
priority = stage_list[-1] if len(stage_list) > 1 else 0
try:
priority = float(priority)
except ValueError:
logger.warn('Task %s has non numeric priority "%s", set to 0', task,
priority)
priority = 0
return stage, priority
def POST(self):
data = self.checked_data()
# TODO: activate and save status
task_manager = DownloadReleaseTaskManager(data['release_id'])
try:
task = task_manager.execute()
except Exception as exc:
logger.warn(u'DownloadReleaseHandler: error while execution'
' deploy task: {0}'.format(exc.message))
raise web.badrequest(exc.message)
return TaskHandler.render(task)
def validate_update(cls, data, instance=None):
data = cls.validate_json(data)
if data.get("master_node_uid"):
logger.warn(
"Changing of master node uid is not allowed. "
"It will be ignored", )
cls.validate_schema(data, schema=MasterNodeSettings.schema)
return data
def DELETE(self, cluster_id):
cluster = self.get_object_or_404(Cluster, cluster_id)
task_manager = ClusterDeletionManager(cluster_id=cluster.id)
try:
logger.debug('Trying to execute cluster deletion task')
task = task_manager.execute()
except Exception as e:
logger.warn('Error while execution '
'cluster deletion task: %s' % str(e))
logger.warn(traceback.format_exc())
raise web.badrequest(str(e))
raise web.webapi.HTTPError(status="202 Accepted", data="{}")
def _get_task_stage_and_priority(task):
stage_list = task['stage'].split('/')
stage = stage_list[0]
priority = stage_list[-1] if len(stage_list) > 1 else 0
try:
priority = float(priority)
except ValueError:
logger.warn(
'Task %s has non numeric priority "%s", set to 0',
task, priority)
priority = 0
return stage, priority
def check_bond_slaves_speeds(self):
"""check bond slaves speeds are equal"""
for node in self.cluster.nodes:
for bond in node.bond_interfaces:
slaves_speed = set(
[slave.current_speed for slave in bond.slaves])
if len(slaves_speed) != 1 or slaves_speed.pop() is None:
warn_msg = u"Node '{0}': interface '{1}' slave NICs " \
u"have different or unrecognized speeds". \
format(node.name, bond.name)
logger.warn(warn_msg)
self.err_msgs.append(warn_msg)
def update_if_exist(cls, task_id, node_id, deployment_graph_task_name,
status, custom):
deployment_history = cls.find_history(task_id, node_id,
deployment_graph_task_name)
if not deployment_history:
logger.warn("Failed to find task in history for transaction id %s"
", node_id %s and deployment_graph_task_name %s",
task_id, node_id, deployment_graph_task_name)
return
getattr(cls, 'to_{0}'.format(status))(deployment_history)
def validate_update(cls, data, instance=None):
data = cls.validate_json(data)
if data.get("master_node_uid"):
logger.warn(
"Changing of master node uid is not allowed. "
"It will be ignored",
)
cls.validate_schema(data, schema=master_node_settings.schema)
return data
def check_bond_slaves_speeds(self):
"""check bond slaves speeds are equal"""
for node in self.cluster.nodes:
for bond in node.bond_interfaces:
slaves_speed = set(
[slave.current_speed for slave in bond.slaves])
if len(slaves_speed) != 1 or slaves_speed.pop() is None:
warn_msg = u"Node '{0}': interface '{1}' slave NICs " \
u"have different or unrecognized speeds". \
format(node.name, bond.name)
logger.warn(warn_msg)
self.err_msgs.append(warn_msg)
def get_postfix(task):
stage_list = task['stage'].split('/')
postfix = stage_list[-1] if len(stage_list) > 1 else 0
try:
postfix = float(postfix)
except ValueError:
logger.warn('Task %s has non numeric postfix "%s", set to 0',
task, postfix)
postfix = 0
return postfix
def provision_resp(cls, **kwargs):
logger.info("RPC method provision_resp received: %s" %
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
#如果返回error,那么将没有返回nodes参数
#导致如果返回安装失败,那么前台node状态将不会更新
task = objects.Task.get_by_uuid(task_uuid,
fail_if_not_found=True,
lock_for_update=True)
# lock nodes for updating
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
[n['uid'] for n in nodes],
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
for node in nodes:
uid = node.get('uid')
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warn('Node with uid "{0}" not found'.format(uid))
continue
if node.get('status') == 'error':
node_db.status = 'error'
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
elif node.get('status') == 'provisioned':
node_db.status = node.get('status')
node_db.progress = node.get('progress')
node_db.power_ip = node_db.ip
logger.info("change the node {0} new power_ip is {1}".format(
node_db.id, node_db.power_ip))
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_provisioning_task_progress(task)
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task_uuid, nodes)
def create(cls, task, tasks_graph):
entries = []
for node_id in tasks_graph:
for graph_task in tasks_graph[node_id]:
if not graph_task.get('id'):
logger.warn("Task name missing. Ignoring %s", graph_task)
continue
entries.append(cls.single.model(
task_id=task.id,
node_id=node_id,
deployment_graph_task_name=graph_task['id']))
db().bulk_save_objects(entries)
def get_postfix(task):
stage_list = task['stage'].split('/')
postfix = stage_list[-1] if len(stage_list) > 1 else 0
try:
postfix = float(postfix)
except ValueError:
logger.warn(
'Task %s has non numeric postfix "%s", set to 0',
task, postfix)
postfix = 0
return postfix
def get_node_spaces(node):
"""Helper for retrieving node volumes.
If spaces don't defained for role, will be used
partitioning for role `other`.
Sets key `_allocate_size` which used only for internal calculation
and not used in partitioning system.
修改成没有集群角色一样可以安装,有集群则优先集群
"""
node_spaces = []
if node.cluster:
role_mapping = node.cluster.release.volumes_metadata[
'volumes_roles_mapping']
elif node.release:
role_mapping =node.release.volumes_metadata[
'volumes_roles_mapping']
else:
role_mapping=defaultrelease.getRelease().volumes_metadata[
'volumes_roles_mapping']
# TODO(dshulyak)
# This logic should go to openstack.yaml (or other template)
# when it will be extended with flexible template engine
modify_volumes_hook(role_mapping, node)
if node.cluster:
all_spaces = node.cluster.release.volumes_metadata['volumes']
elif node.release:
all_spaces = node.release.volumes_metadata['volumes']
else:
all_spaces=defaultrelease.getRelease().volumes_metadata['volumes']
for role in node.all_roles:
if not role_mapping.get(role):
continue
volumes = role_mapping[role]
for volume in volumes:
space = find_space_by_id(all_spaces, volume['id'])
if space not in node_spaces:
space['_allocate_size'] = get_allocate_size(node, volume)
node_spaces.append(space)
# Use role `other`
if not node_spaces:
logger.warn('Cannot find volumes for node: %s assigning default '
'volumes' % (node.full_name))
for volume in role_mapping['other']:
space = find_space_by_id(all_spaces, volume['id'])
space['_allocate_size'] = get_allocate_size(node, volume)
node_spaces.append(space)
return node_spaces
def handle_task(self, cluster, **kwargs):
nodes = self.get_nodes(cluster)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes, **kwargs)
except Exception as exc:
logger.warn(
u'Cannot execute %s task nodes: %s',
task_manager.__class__.__name__, traceback.format_exc())
raise self.http(400, message=six.text_type(exc))
self.raise_task(task)
def handle_task(self, cluster, **kwargs):
nodes = self.get_nodes(cluster)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes, **kwargs)
except Exception as exc:
logger.warn(u'Cannot execute %s task nodes: %s',
task_manager.__class__.__name__,
traceback.format_exc())
raise self.http(400, message=six.text_type(exc))
raise self.http(202, objects.Task.to_json(task))
def consume_msg(self, body, msg):
callback = getattr(self.receiver, body["method"])
try:
callback(**body["args"])
db().commit()
except errors.CannotFindTask as e:
logger.warn(str(e))
db().rollback()
except Exception:
logger.error(traceback.format_exc())
db().rollback()
finally:
msg.ack()
db().expire_all()
def provision_resp(cls, **kwargs):
logger.info(
"RPC method provision_resp received: %s" %
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
#如果返回error,那么将没有返回nodes参数
#导致如果返回安装失败,那么前台node状态将不会更新
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True,
lock_for_update=True
)
# lock nodes for updating
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
[n['uid'] for n in nodes],
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
for node in nodes:
uid = node.get('uid')
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warn('Node with uid "{0}" not found'.format(uid))
continue
if node.get('status') == 'error':
node_db.status = 'error'
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_provisioning_task_progress(task)
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task_uuid, nodes)
def consume_msg(self, body, msg):
callback = getattr(self.receiver, body["method"])
try:
callback(**body["args"])
db().commit()
except errors.CannotFindTask as e:
logger.warn(str(e))
db().rollback()
except Exception:
logger.error(traceback.format_exc())
db().rollback()
finally:
msg.ack()
db().expire_all()
def remove_nodes_resp(cls, **kwargs):
logger.info("RPC method remove_nodes_resp received: %s" % json.dumps(kwargs))
task_uuid = kwargs.get("task_uuid")
nodes = kwargs.get("nodes") or []
error_nodes = kwargs.get("error_nodes") or []
inaccessible_nodes = kwargs.get("inaccessible_nodes") or []
error_msg = kwargs.get("error")
status = kwargs.get("status")
progress = kwargs.get("progress")
for node in nodes:
node_db = db().query(Node).get(node["uid"])
if not node_db:
logger.error(u"Failed to delete node '%s': node doesn't exist", str(node))
break
db().delete(node_db)
for node in inaccessible_nodes:
# Nodes which not answered by rpc just removed from db
node_db = db().query(Node).get(node["uid"])
if node_db:
logger.warn(u"Node %s not answered by RPC, removing from db", node_db.human_readable_name)
db().delete(node_db)
for node in error_nodes:
node_db = db().query(Node).get(node["uid"])
if not node_db:
logger.error(u"Failed to delete node '%s' marked as error from Naily:" " node doesn't exist", str(node))
break
node_db.pending_deletion = False
node_db.status = "error"
db().add(node_db)
node["name"] = node_db.name
db().commit()
success_msg = u"No nodes were removed"
err_msg = u"No errors occurred"
if nodes:
success_msg = u"Successfully removed {0} node(s)".format(len(nodes))
notifier.notify("done", success_msg)
if error_nodes:
err_msg = u"Failed to remove {0} node(s): {1}".format(
len(error_nodes), ", ".join([n.get("name") or "ID: {0}".format(n["uid"]) for n in error_nodes])
)
notifier.notify("error", err_msg)
if not error_msg:
error_msg = ". ".join([success_msg, err_msg])
TaskHelper.update_task_status(task_uuid, status, progress, error_msg)
def handle_task(self, cluster, **kwargs):
nodes = self.get_nodes(cluster)
if nodes:
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes_to_provision_deploy=nodes,
**kwargs)
except Exception as exc:
logger.warn(u'Cannot execute %s task nodes: %s',
task_manager.__class__.__name__,
traceback.format_exc())
raise self.http(400, six.text_type(exc))
self.raise_task(task)
else:
raise self.http(400, "No VMs to spawn")
def PUT(self, cluster_id):
cluster = self.get_object_or_404(
Cluster,
cluster_id,
log_404=("warning", "Error: there is no cluster "
"with id '{0}' in DB.".format(cluster_id)))
task_manager = DeploymentTaskManager(cluster_id=cluster.id)
try:
task = task_manager.execute()
except Exception as exc:
logger.warn(u'ClusterChangesHandler: error while execution'
' deploy task: {0}'.format(exc.message))
raise web.badrequest(exc.message)
return TaskHandler.render(task)
def handle_task(self, cluster, **kwargs):
nodes = self.get_nodes(cluster)
if nodes:
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes_to_provision_deploy=nodes,
**kwargs)
except Exception as exc:
logger.warn(
u'Cannot execute %s task nodes: %s',
task_manager.__class__.__name__, traceback.format_exc())
raise self.http(400, six.text_type(exc))
self.raise_task(task)
else:
raise self.http(400, "No VMs to spawn")
def DELETE(self, cluster_id):
cluster = self.get_object_or_404(Cluster, cluster_id)
task_manager = ClusterDeletionManager(cluster_id=cluster.id)
try:
logger.debug('Trying to execute cluster deletion task')
task = task_manager.execute()
except Exception as e:
logger.warn('Error while execution '
'cluster deletion task: %s' % str(e))
logger.warn(traceback.format_exc())
raise web.badrequest(str(e))
raise web.webapi.HTTPError(
status="202 Accepted",
data="{}"
)
def get_uids_for_tasks(self, tasks):
uids = []
for task in tasks:
if isinstance(task['role'], list):
for node in self.nodes:
required_for_node = set(task['role']) & set(node.all_roles)
if required_for_node:
uids.append(node.uid)
elif task['role'] == '*':
uids.extend([n.uid for n in self.nodes])
else:
logger.warn(
'Wrong task format, `role` should be a list or "*": %s',
task)
return list(set(uids))
def get_uids_for_tasks(self, tasks):
uids = []
for task in tasks:
if isinstance(task['role'], list):
for node in self.nodes:
required_for_node = set(task['role']) & set(node.all_roles)
if required_for_node:
uids.append(node.uid)
elif task['role'] == '*':
uids.extend([n.uid for n in self.nodes])
else:
logger.warn(
'Wrong task format, `role` should be a list or "*": %s',
task)
return list(set(uids))
def DELETE(self, cluster_id):
""":returns: {}
:http: * 202 (cluster deletion process launched)
* 400 (failed to execute cluster deletion process)
* 404 (cluster not found in db)
"""
cluster = self.get_object_or_404(Cluster, cluster_id)
task_manager = ClusterDeletionManager(cluster_id=cluster.id)
try:
logger.debug('Trying to execute cluster deletion task')
task_manager.execute()
except Exception as e:
logger.warn('Error while execution '
'cluster deletion task: %s' % str(e))
logger.warn(traceback.format_exc())
raise web.badrequest(str(e))
raise web.webapi.HTTPError(status="202 Accepted", data="{}")
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 200 (task successfully executed)
* 404 (cluster or nodes not found in db)
* 400 (failed to execute task)
"""
cluster = self.get_object_or_404(Cluster, cluster_id)
nodes = self.get_nodes(cluster)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes)
except Exception as exc:
logger.warn(u'Cannot execute {0} task nodes: {1}'.format(
task_manager.__class__.__name__, traceback.format_exc()))
raise web.badrequest(str(exc))
return TaskHandler.render(task)
def get_uids_for_tasks(nodes, tasks):
"""Return node uids where particular tasks should be executed
:param nodes: list of Node db objects
:param tasks: list of dicts
:returns: list of strings
"""
roles = []
for task in tasks:
if task['role'] == consts.ALL_ROLES:
return get_uids_for_roles(nodes, consts.ALL_ROLES)
elif isinstance(task['role'], list):
roles.extend(task['role'])
else:
logger.warn(
'Wrong roles format, `roles` should be a list or "*" in %s',
task)
return get_uids_for_roles(nodes, roles)
def DELETE(self, obj_id):
""":returns: {}
:http: * 202 (cluster deletion process launched)
* 400 (failed to execute cluster deletion process)
* 404 (cluster not found in db)
"""
cluster = self.get_object_or_404(self.single, obj_id)
task_manager = ClusterDeletionManager(cluster_id=cluster.id)
try:
logger.debug('Trying to execute cluster deletion task')
task_manager.execute()
except Exception as e:
logger.warn('Error while execution '
'cluster deletion task: %s' % str(e))
logger.warn(traceback.format_exc())
raise self.http(400, str(e))
raise self.http(202, '{}')
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 200 (task successfully executed)
* 404 (cluster or nodes not found in db)
* 400 (failed to execute task)
"""
cluster = self.get_object_or_404(objects.Cluster, cluster_id)
nodes = self.get_nodes(cluster)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes)
except Exception as exc:
logger.warn(u'Cannot execute {0} task nodes: {1}'.format(
task_manager.__class__.__name__, traceback.format_exc()))
raise self.http(400, message=str(exc))
raise self.http(202, objects.Task.to_json(task))
