def test_recalculate_deployment_task_progress(self):
cluster = self.create_env([{
'roles': ['controller'],
'status': 'provisioned',
'progress': 100
}, {
'roles': ['compute'],
'status': 'deploying',
'progress': 100
}, {
'roles': ['compute'],
'status': 'ready',
'progress': 0
}, {
'roles': ['compute'],
'status': 'discover',
'progress': 0
}])
task = Task(name='deploy', cluster_id=cluster.id)
self.db.add(task)
self.db.commit()
progress = TaskHelper.recalculate_deployment_task_progress(task)
self.assertEquals(progress, 25)
def test_recalculate_deployment_task_progress(self):
cluster = self.create_env([
{'roles': ['controller'],
'status': 'provisioned',
'progress': 100},
{'roles': ['compute'],
'status': 'deploying',
'progress': 100},
{'roles': ['compute'],
'status': 'ready',
'progress': 0},
{'roles': ['compute'],
'status': 'discover',
'progress': 0}])
task = Task(name='deploy', cluster_id=cluster.id)
self.db.add(task)
self.db.commit()
progress = TaskHelper.recalculate_deployment_task_progress(task)
self.assertEquals(progress, 25)
def deploy_resp(cls, **kwargs):
logger.info(
"RPC method deploy_resp received: %s" %
jsonutils.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True,
)
# locking all cluster tasks
objects.TaskCollection.lock_cluster_tasks(task.cluster_id)
# lock cluster
objects.Cluster.get_by_uid(
task.cluster_id,
fail_if_not_found=True,
lock_for_update=True
)
if not status:
status = task.status
# for deployment we need just to pop
master = next((
n for n in nodes if n['uid'] == consts.MASTER_ROLE), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code
if master:
nodes.remove(master)
# if there no node except master - then just skip updating
# nodes status, for the task itself astute will send
# message with descriptive error
if nodes:
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
[n['uid'] for n in nodes],
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
# First of all, let's update nodes in database
for node in nodes:
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']
)
)
continue
update_fields = (
'error_msg',
'error_type',
'status',
'progress',
'online'
)
for param in update_fields:
if param in node:
logger.debug(
u"Updating node {0} - set {1} to {2}".format(
node['uid'],
param,
node[param]
)
)
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name,
node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid
)
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# full error will be provided in next astute message
if master.get('status') == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
# Let's check the whole task status
if status == consts.TASK_STATUSES.error:
cls._error_action(task, status, progress, message)
elif status == consts.TASK_STATUSES.ready:
cls._success_action(task, status, progress)
else:
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs):
logger.info(
"RPC method deploy_resp received: %s" %
jsonutils.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True,
lock_for_update=True
)
# lock cluster for updating so it can't be deleted
objects.Cluster.get_by_uid(
task.cluster_id,
fail_if_not_found=True,
lock_for_update=True
)
if not status:
status = task.status
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
[n['uid'] for n in nodes],
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
# First of all, let's update nodes in database
for node in nodes:
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']
)
)
continue
update_fields = (
'error_msg',
'error_type',
'status',
'progress',
'online'
)
for param in update_fields:
if param in node:
logger.debug(
u"Updating node {0} - set {1} to {2}".format(
node['uid'],
param,
node[param]
)
)
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name,
node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid
)
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# Let's check the whole task status
if status in ('error',):
cls._error_action(task, status, progress, message)
elif status in ('ready',):
cls._success_action(task, status, progress)
else:
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
def deploy_resp(cls, **kwargs):
logger.info(
"RPC method deploy_resp received: %s" %
jsonutils.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True
)
# lock cluster
objects.Cluster.get_by_uid(
task.cluster_id,
fail_if_not_found=True,
lock_for_update=True
)
if not status:
status = task.status
# for deployment we need just to pop
# if there no node except master - then just skip updating
# nodes status, for the task itself astute will send
# message with descriptive error
nodes_by_id = {str(n['uid']): n for n in nodes}
master = nodes_by_id.pop(consts.MASTER_NODE_UID, {})
if nodes_by_id:
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
nodes_by_id,
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
db_nodes = objects.NodeCollection.lock_for_update(q_nodes).all()
else:
db_nodes = []
# First of all, let's update nodes in database
for node_db in db_nodes:
node = nodes_by_id.pop(node_db.uid)
update_fields = (
'error_msg',
'error_type',
'status',
'progress',
'online'
)
for param in update_fields:
if param in node:
logger.debug("Updating node %s - set %s to %s",
node['uid'], param, node[param])
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
consts.NOTIFICATION_TOPICS.error,
u"Failed to {0} node '{1}': {2}".format(
consts.TASK_NAMES.deploy,
node_db.name,
node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid
)
db().flush()
if nodes_by_id:
logger.warning("The following nodes is not found: %s",
",".join(sorted(nodes_by_id)))
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# full error will be provided in next astute message
if master.get('status') == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
cls._update_task_status(task, status, progress, message, db_nodes)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs):
logger.info("RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs))
task_uuid = kwargs.get("task_uuid")
nodes = kwargs.get("nodes") or []
message = kwargs.get("error")
status = kwargs.get("status")
progress = kwargs.get("progress")
task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
# lock cluster
objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True)
if not status:
status = task.status
# for deployment we need just to pop
master = next((n for n in nodes if n["uid"] == consts.MASTER_NODE_UID), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code
if master:
nodes.remove(master)
# if there no node except master - then just skip updating
# nodes status, for the task itself astute will send
# message with descriptive error
if nodes:
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(None, [n["uid"] for n in nodes])
q_nodes = objects.NodeCollection.order_by(q_nodes, "id")
objects.NodeCollection.lock_for_update(q_nodes).all()
# First of all, let's update nodes in database
for node in nodes:
node_db = objects.Node.get_by_uid(node["uid"])
if not node_db:
logger.warning(u"No node found with uid '{0}' - nothing changed".format(node["uid"]))
continue
update_fields = ("error_msg", "error_type", "status", "progress", "online")
for param in update_fields:
if param in node:
logger.debug(u"Updating node {0} - set {1} to {2}".format(node["uid"], param, node[param]))
setattr(node_db, param, node[param])
if param == "progress" and node.get("status") == "error" or node.get("online") is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get("online") is False and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
consts.NOTIFICATION_TOPICS.error,
u"Failed to {0} node '{1}': {2}".format(
consts.TASK_NAMES.deploy, node_db.name, node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node["uid"],
task_uuid=task_uuid,
)
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# full error will be provided in next astute message
if master.get("status") == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
cls._update_task_status(task, status, progress, message)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs):
logger.info(
"RPC method deploy_resp received: %s" %
jsonutils.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True
)
# lock cluster
objects.Cluster.get_by_uid(
task.cluster_id,
fail_if_not_found=True,
lock_for_update=True
)
if not status:
status = task.status
# for deployment we need just to pop
master = next((
n for n in nodes if n['uid'] == consts.MASTER_NODE_UID), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code
if master:
nodes.remove(master)
# if there no node except master - then just skip updating
# nodes status, for the task itself astute will send
# message with descriptive error
if nodes:
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
[n['uid'] for n in nodes],
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
objects.NodeCollection.lock_for_update(q_nodes).all()
# First of all, let's update nodes in database
for node in nodes:
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']
)
)
continue
update_fields = (
'error_msg',
'error_type',
'status',
'progress',
'online'
)
for param in update_fields:
if param in node:
logger.debug(
u"Updating node {0} - set {1} to {2}".format(
node['uid'],
param,
node[param]
)
)
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
consts.NOTIFICATION_TOPICS.error,
u"Failed to {0} node '{1}': {2}".format(
consts.TASK_NAMES.deploy,
node_db.name,
node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid
)
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# full error will be provided in next astute message
if master.get('status') == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
cls._update_task_status(task, status, progress, message)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs):
logger.info("RPC method deploy_resp received: %s" % json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(task_uuid,
fail_if_not_found=True,
lock_for_update=True)
# lock cluster for updating so it can't be deleted
objects.Cluster.get_by_uid(task.cluster_id,
fail_if_not_found=True,
lock_for_update=True)
if not status:
status = task.status
# lock nodes for updating so they can't be deleted
list(
objects.NodeCollection.lock_for_update(
objects.NodeCollection.get_by_id_list(
None, [n['uid'] for n in nodes])))
# First of all, let's update nodes in database
for node in nodes:
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']))
continue
update_fields = ('error_msg', 'error_type', 'status', 'progress',
'online')
for param in update_fields:
if param in node:
logger.debug(u"Updating node {0} - set {1} to {2}".format(
node['uid'], param, node[param]))
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name, node_db.error_msg
or "Unknown error"),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid)
db().add(node_db)
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# Let's check the whole task status
if status in ('error', ):
cls._error_action(task, status, progress, message)
elif status in ('ready', ):
cls._success_action(task, status, progress)
else:
TaskHelper.update_task_status(task.uuid, status, progress, message)
def deploy_resp(cls, **kwargs):
logger.info("RPC method deploy_resp received: %s" %
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
# lock cluster
objects.Cluster.get_by_uid(task.cluster_id,
fail_if_not_found=True,
lock_for_update=True)
if not status:
status = task.status
# for deployment we need just to pop
# if there no node except master - then just skip updating
# nodes status, for the task itself astute will send
# message with descriptive error
nodes_by_id = {str(n['uid']): n for n in nodes}
master = nodes_by_id.pop(consts.MASTER_NODE_UID, {})
nodes_by_id.pop('None', {})
if nodes_by_id:
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(
None,
nodes_by_id,
)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
db_nodes = objects.NodeCollection.lock_for_update(q_nodes).all()
else:
db_nodes = []
# First of all, let's update nodes in database
for node_db in db_nodes:
node = nodes_by_id.pop(node_db.uid)
update_fields = ('error_msg', 'error_type', 'status', 'progress',
'online')
for param in update_fields:
if param in node:
logger.debug("Updating node %s - set %s to %s",
node['uid'], param, node[param])
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
consts.NOTIFICATION_TOPICS.error,
u"Failed to {0} node '{1}': {2}".format(
consts.TASK_NAMES.deploy, node_db.name,
node_db.error_msg or "Unknown error"),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid)
if nodes_by_id:
logger.warning("The following nodes is not found: %s",
",".join(sorted(nodes_by_id)))
for node in nodes:
if node.get('deployment_graph_task_name') \
and node.get('task_status'):
objects.DeploymentHistory.update_if_exist(
task.id, node['uid'], node['deployment_graph_task_name'],
node['task_status'], node.get('custom'))
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# full error will be provided in next astute message
if master.get('status') == consts.TASK_STATUSES.error:
status = consts.TASK_STATUSES.error
cls._update_task_status(task, status, progress, message, db_nodes)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs):
logger.info("RPC method deploy_resp received: %s" % json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = get_task_by_uuid(task_uuid)
if not task:
# No task found - nothing to do here, returning
logger.warning(
u"No task with uuid '{0}'' found - nothing changed".format(
task_uuid))
return
if not status:
status = task.status
# First of all, let's update nodes in database
for node in nodes:
node_db = db().query(Node).get(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']))
continue
update_fields = ('error_msg', 'error_type', 'status', 'progress',
'online')
for param in update_fields:
if param in node:
logger.debug(u"Updating node {0} - set {1} to {2}".format(
node['uid'], param, node[param]))
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name, node_db.error_msg
or "Unknown error"),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid)
db().add(node_db)
db().commit()
# We should calculate task progress by nodes info
task = get_task_by_uuid(task_uuid)
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# Let's check the whole task status
if status in ('error', ):
cls._error_action(task, status, progress, message)
elif status in ('ready', ):
cls._success_action(task, status, progress)
else:
TaskHelper.update_task_status(task.uuid, status, progress, message)
def deploy_resp(cls, **kwargs):
logger.info(
"RPC method deploy_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = get_task_by_uuid(task_uuid)
if not task:
# No task found - nothing to do here, returning
logger.warning(
u"No task with uuid '{0}'' found - nothing changed".format(
task_uuid
)
)
return
if not status:
status = task.status
# First of all, let's update nodes in database
for node in nodes:
node_db = db().query(Node).get(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']
)
)
continue
update_fields = (
'error_msg',
'error_type',
'status',
'progress',
'online'
)
for param in update_fields:
if param in node:
logger.debug(
u"Updating node {0} - set {1} to {2}".format(
node['uid'],
param,
node[param]
)
)
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name,
node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid
)
db().add(node_db)
db().commit()
# We should calculate task progress by nodes info
task = get_task_by_uuid(task_uuid)
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# Let's check the whole task status
if status in ('error',):
cls._error_action(task, status, progress, message)
elif status in ('ready',):
cls._success_action(task, status, progress)
else:
TaskHelper.update_task_status(task.uuid, status, progress, message)
def start_stop_resp(cls, **kwargs):
logger.info("RPC method start_stop_resp received: %s" % jsonutils.dumps(kwargs))
task_uuid = kwargs.get("task_uuid")
nodes = kwargs.get("nodes") or []
message = kwargs.get("error")
status = kwargs.get("status")
progress = kwargs.get("progress")
task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
# locking all cluster tasks
objects.TaskCollection.lock_cluster_tasks(task.cluster_id)
# lock cluster
objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True)
if not status:
status = task.status
# lock nodes for updating so they can't be deleted
q_nodes = objects.NodeCollection.filter_by_id_list(None, [n["uid"] for n in nodes])
q_nodes = objects.NodeCollection.order_by(q_nodes, "id")
objects.NodeCollection.lock_for_update(q_nodes).all()
# First of all, let's update nodes in database
for node in nodes:
node_db = objects.Node.get_by_uid(node["uid"])
if not node_db:
logger.warning(u"No node found with uid '{0}' - nothing changed".format(node["uid"]))
continue
update_fields = ("error_msg", "error_type", "status", "progress", "online")
for param in update_fields:
if param in node:
logger.debug(u"Updating node {0} - set {1} to {2}".format(node["uid"], param, node[param]))
setattr(node_db, param, node[param])
if param == "progress" and node.get("status") == "error" or node.get("online") is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get("online") is False and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name, node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node["uid"],
task_uuid=task_uuid,
)
db().flush()
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# Let's check the whole task status
if status in ("error",):
cls._error_start_action(task, status, progress, message)
elif status in ("ready",):
cls._success_start_action(task, status, progress)
else:
data = {"status": status, "progress": progress, "message": message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task_uuid, nodes)
