def test_recalculate_deployment_task_progress(self): cluster = self.create_env([{ 'roles': ['controller'], 'status': 'provisioned', 'progress': 100 }, { 'roles': ['compute'], 'status': 'deploying', 'progress': 100 }, { 'roles': ['compute'], 'status': 'ready', 'progress': 0 }, { 'roles': ['compute'], 'status': 'discover', 'progress': 0 }]) task = Task(name='deploy', cluster_id=cluster.id) self.db.add(task) self.db.commit() progress = TaskHelper.recalculate_deployment_task_progress(task) self.assertEquals(progress, 25)
def test_recalculate_deployment_task_progress(self): cluster = self.create_env([ {'roles': ['controller'], 'status': 'provisioned', 'progress': 100}, {'roles': ['compute'], 'status': 'deploying', 'progress': 100}, {'roles': ['compute'], 'status': 'ready', 'progress': 0}, {'roles': ['compute'], 'status': 'discover', 'progress': 0}]) task = Task(name='deploy', cluster_id=cluster.id) self.db.add(task) self.db.commit() progress = TaskHelper.recalculate_deployment_task_progress(task) self.assertEquals(progress, 25)
def deploy_resp(cls, **kwargs): logger.info( "RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True, ) # locking all cluster tasks objects.TaskCollection.lock_cluster_tasks(task.cluster_id) # lock cluster objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True ) if not status: status = task.status # for deployment we need just to pop master = next(( n for n in nodes if n['uid'] == consts.MASTER_ROLE), {}) # we should remove master node from the nodes since it requires # special handling and won't work with old code if master: nodes.remove(master) # if there no node except master - then just skip updating # nodes status, for the task itself astute will send # message with descriptive error if nodes: # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list( None, [n['uid'] for n in nodes], ) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') objects.NodeCollection.lock_for_update(q_nodes).all() # First of all, let's update nodes in database for node in nodes: node_db = objects.Node.get_by_uid(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'] ) ) continue update_fields = ( 'error_msg', 'error_type', 'status', 'progress', 'online' ) for param in update_fields: if param in node: logger.debug( u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param] ) ) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid ) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # full error will be provided in next astute message if master.get('status') == consts.TASK_STATUSES.error: status = consts.TASK_STATUSES.error # Let's check the whole task status if status == consts.TASK_STATUSES.error: cls._error_action(task, status, progress, message) elif status == consts.TASK_STATUSES.ready: cls._success_action(task, status, progress) else: data = {'status': status, 'progress': progress, 'message': message} objects.Task.update(task, data) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs): logger.info( "RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True, lock_for_update=True ) # lock cluster for updating so it can't be deleted objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True ) if not status: status = task.status # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list( None, [n['uid'] for n in nodes], ) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') objects.NodeCollection.lock_for_update(q_nodes).all() # First of all, let's update nodes in database for node in nodes: node_db = objects.Node.get_by_uid(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'] ) ) continue update_fields = ( 'error_msg', 'error_type', 'status', 'progress', 'online' ) for param in update_fields: if param in node: logger.debug( u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param] ) ) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid ) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # Let's check the whole task status if status in ('error',): cls._error_action(task, status, progress, message) elif status in ('ready',): cls._success_action(task, status, progress) else: data = {'status': status, 'progress': progress, 'message': message} objects.Task.update(task, data)
def deploy_resp(cls, **kwargs): logger.info( "RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True ) # lock cluster objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True ) if not status: status = task.status # for deployment we need just to pop # if there no node except master - then just skip updating # nodes status, for the task itself astute will send # message with descriptive error nodes_by_id = {str(n['uid']): n for n in nodes} master = nodes_by_id.pop(consts.MASTER_NODE_UID, {}) if nodes_by_id: # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list( None, nodes_by_id, ) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') db_nodes = objects.NodeCollection.lock_for_update(q_nodes).all() else: db_nodes = [] # First of all, let's update nodes in database for node_db in db_nodes: node = nodes_by_id.pop(node_db.uid) update_fields = ( 'error_msg', 'error_type', 'status', 'progress', 'online' ) for param in update_fields: if param in node: logger.debug("Updating node %s - set %s to %s", node['uid'], param, node[param]) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( consts.NOTIFICATION_TOPICS.error, u"Failed to {0} node '{1}': {2}".format( consts.TASK_NAMES.deploy, node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid ) db().flush() if nodes_by_id: logger.warning("The following nodes is not found: %s", ",".join(sorted(nodes_by_id))) if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # full error will be provided in next astute message if master.get('status') == consts.TASK_STATUSES.error: status = consts.TASK_STATUSES.error cls._update_task_status(task, status, progress, message, db_nodes) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs): logger.info("RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs)) task_uuid = kwargs.get("task_uuid") nodes = kwargs.get("nodes") or [] message = kwargs.get("error") status = kwargs.get("status") progress = kwargs.get("progress") task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True) # lock cluster objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True) if not status: status = task.status # for deployment we need just to pop master = next((n for n in nodes if n["uid"] == consts.MASTER_NODE_UID), {}) # we should remove master node from the nodes since it requires # special handling and won't work with old code if master: nodes.remove(master) # if there no node except master - then just skip updating # nodes status, for the task itself astute will send # message with descriptive error if nodes: # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list(None, [n["uid"] for n in nodes]) q_nodes = objects.NodeCollection.order_by(q_nodes, "id") objects.NodeCollection.lock_for_update(q_nodes).all() # First of all, let's update nodes in database for node in nodes: node_db = objects.Node.get_by_uid(node["uid"]) if not node_db: logger.warning(u"No node found with uid '{0}' - nothing changed".format(node["uid"])) continue update_fields = ("error_msg", "error_type", "status", "progress", "online") for param in update_fields: if param in node: logger.debug(u"Updating node {0} - set {1} to {2}".format(node["uid"], param, node[param])) setattr(node_db, param, node[param]) if param == "progress" and node.get("status") == "error" or node.get("online") is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get("online") is False and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( consts.NOTIFICATION_TOPICS.error, u"Failed to {0} node '{1}': {2}".format( consts.TASK_NAMES.deploy, node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node["uid"], task_uuid=task_uuid, ) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # full error will be provided in next astute message if master.get("status") == consts.TASK_STATUSES.error: status = consts.TASK_STATUSES.error cls._update_task_status(task, status, progress, message) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs): logger.info( "RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True ) # lock cluster objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True ) if not status: status = task.status # for deployment we need just to pop master = next(( n for n in nodes if n['uid'] == consts.MASTER_NODE_UID), {}) # we should remove master node from the nodes since it requires # special handling and won't work with old code if master: nodes.remove(master) # if there no node except master - then just skip updating # nodes status, for the task itself astute will send # message with descriptive error if nodes: # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list( None, [n['uid'] for n in nodes], ) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') objects.NodeCollection.lock_for_update(q_nodes).all() # First of all, let's update nodes in database for node in nodes: node_db = objects.Node.get_by_uid(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'] ) ) continue update_fields = ( 'error_msg', 'error_type', 'status', 'progress', 'online' ) for param in update_fields: if param in node: logger.debug( u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param] ) ) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( consts.NOTIFICATION_TOPICS.error, u"Failed to {0} node '{1}': {2}".format( consts.TASK_NAMES.deploy, node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid ) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # full error will be provided in next astute message if master.get('status') == consts.TASK_STATUSES.error: status = consts.TASK_STATUSES.error cls._update_task_status(task, status, progress, message) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs): logger.info("RPC method deploy_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True, lock_for_update=True) # lock cluster for updating so it can't be deleted objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True) if not status: status = task.status # lock nodes for updating so they can't be deleted list( objects.NodeCollection.lock_for_update( objects.NodeCollection.get_by_id_list( None, [n['uid'] for n in nodes]))) # First of all, let's update nodes in database for node in nodes: node_db = objects.Node.get_by_uid(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'])) continue update_fields = ('error_msg', 'error_type', 'status', 'progress', 'online') for param in update_fields: if param in node: logger.debug(u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param])) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error"), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid) db().add(node_db) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # Let's check the whole task status if status in ('error', ): cls._error_action(task, status, progress, message) elif status in ('ready', ): cls._success_action(task, status, progress) else: TaskHelper.update_task_status(task.uuid, status, progress, message)
def deploy_resp(cls, **kwargs): logger.info("RPC method deploy_resp received: %s" % jsonutils.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True) # lock cluster objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True) if not status: status = task.status # for deployment we need just to pop # if there no node except master - then just skip updating # nodes status, for the task itself astute will send # message with descriptive error nodes_by_id = {str(n['uid']): n for n in nodes} master = nodes_by_id.pop(consts.MASTER_NODE_UID, {}) nodes_by_id.pop('None', {}) if nodes_by_id: # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list( None, nodes_by_id, ) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') db_nodes = objects.NodeCollection.lock_for_update(q_nodes).all() else: db_nodes = [] # First of all, let's update nodes in database for node_db in db_nodes: node = nodes_by_id.pop(node_db.uid) update_fields = ('error_msg', 'error_type', 'status', 'progress', 'online') for param in update_fields: if param in node: logger.debug("Updating node %s - set %s to %s", node['uid'], param, node[param]) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( consts.NOTIFICATION_TOPICS.error, u"Failed to {0} node '{1}': {2}".format( consts.TASK_NAMES.deploy, node_db.name, node_db.error_msg or "Unknown error"), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid) if nodes_by_id: logger.warning("The following nodes is not found: %s", ",".join(sorted(nodes_by_id))) for node in nodes: if node.get('deployment_graph_task_name') \ and node.get('task_status'): objects.DeploymentHistory.update_if_exist( task.id, node['uid'], node['deployment_graph_task_name'], node['task_status'], node.get('custom')) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # full error will be provided in next astute message if master.get('status') == consts.TASK_STATUSES.error: status = consts.TASK_STATUSES.error cls._update_task_status(task, status, progress, message, db_nodes) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def deploy_resp(cls, **kwargs): logger.info("RPC method deploy_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = get_task_by_uuid(task_uuid) if not task: # No task found - nothing to do here, returning logger.warning( u"No task with uuid '{0}'' found - nothing changed".format( task_uuid)) return if not status: status = task.status # First of all, let's update nodes in database for node in nodes: node_db = db().query(Node).get(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'])) continue update_fields = ('error_msg', 'error_type', 'status', 'progress', 'online') for param in update_fields: if param in node: logger.debug(u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param])) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error"), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid) db().add(node_db) db().commit() # We should calculate task progress by nodes info task = get_task_by_uuid(task_uuid) if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # Let's check the whole task status if status in ('error', ): cls._error_action(task, status, progress, message) elif status in ('ready', ): cls._success_action(task, status, progress) else: TaskHelper.update_task_status(task.uuid, status, progress, message)
def deploy_resp(cls, **kwargs): logger.info( "RPC method deploy_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = get_task_by_uuid(task_uuid) if not task: # No task found - nothing to do here, returning logger.warning( u"No task with uuid '{0}'' found - nothing changed".format( task_uuid ) ) return if not status: status = task.status # First of all, let's update nodes in database for node in nodes: node_db = db().query(Node).get(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'] ) ) continue update_fields = ( 'error_msg', 'error_type', 'status', 'progress', 'online' ) for param in update_fields: if param in node: logger.debug( u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param] ) ) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid ) db().add(node_db) db().commit() # We should calculate task progress by nodes info task = get_task_by_uuid(task_uuid) if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # Let's check the whole task status if status in ('error',): cls._error_action(task, status, progress, message) elif status in ('ready',): cls._success_action(task, status, progress) else: TaskHelper.update_task_status(task.uuid, status, progress, message)
def start_stop_resp(cls, **kwargs): logger.info("RPC method start_stop_resp received: %s" % jsonutils.dumps(kwargs)) task_uuid = kwargs.get("task_uuid") nodes = kwargs.get("nodes") or [] message = kwargs.get("error") status = kwargs.get("status") progress = kwargs.get("progress") task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True) # locking all cluster tasks objects.TaskCollection.lock_cluster_tasks(task.cluster_id) # lock cluster objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True) if not status: status = task.status # lock nodes for updating so they can't be deleted q_nodes = objects.NodeCollection.filter_by_id_list(None, [n["uid"] for n in nodes]) q_nodes = objects.NodeCollection.order_by(q_nodes, "id") objects.NodeCollection.lock_for_update(q_nodes).all() # First of all, let's update nodes in database for node in nodes: node_db = objects.Node.get_by_uid(node["uid"]) if not node_db: logger.warning(u"No node found with uid '{0}' - nothing changed".format(node["uid"])) continue update_fields = ("error_msg", "error_type", "status", "progress", "online") for param in update_fields: if param in node: logger.debug(u"Updating node {0} - set {1} to {2}".format(node["uid"], param, node[param])) setattr(node_db, param, node[param]) if param == "progress" and node.get("status") == "error" or node.get("online") is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get("online") is False and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node["uid"], task_uuid=task_uuid, ) db().flush() if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # Let's check the whole task status if status in ("error",): cls._error_start_action(task, status, progress, message) elif status in ("ready",): cls._success_start_action(task, status, progress) else: data = {"status": status, "progress": progress, "message": message} objects.Task.update(task, data) cls._update_action_log_entry(status, task_uuid, nodes)