def reset_environment_resp(cls, **kwargs): logger.info("RPC method reset_environment_resp received: %s", jsonutils.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True, lock_for_update=True) # Locking cluster objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True) if status == consts.TASK_STATUSES.ready: # restoring pending changes task.cluster.status = consts.CLUSTER_STATUSES.new objects.Cluster.add_pending_changes( task.cluster, consts.CLUSTER_CHANGES.attributes) objects.Cluster.add_pending_changes( task.cluster, consts.CLUSTER_CHANGES.networks) node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)] q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids) q_nodes = objects.NodeCollection.filter_by( q_nodes, cluster_id=task.cluster_id) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') # locking Nodes for update update_nodes = objects.NodeCollection.lock_for_update( q_nodes).all() for node in update_nodes: logs_utils.delete_node_logs(node) objects.Node.reset_to_discover(node) if ia_nodes: cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"environment resetting") message = (u"Environment '{0}' " u"was successfully reset".format(task.cluster.name or task.cluster_id)) notifier.notify("done", message, task.cluster_id) data = {'status': status, 'progress': progress, 'message': message} objects.Task.update(task, data) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def test_delete_node_no_existing_logs(self, _): """Only checks whether errors are passing silently. That's why there's no assertions, just expecting no errors. """ prefix = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, prefix) cluster = self.create_env([{'roles': ['controller']}]) node = cluster.nodes[0] logs_utils.delete_node_logs(node, prefix)
def test_delete_node_no_existing_logs(self, _): """Only checks whether errors are passing silently. That's why there's no assertions, just expecting no errors. """ prefix = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, prefix) cluster = self.create_env([{'roles': ['controller']}]) node = cluster.nodes[0] logs_utils.delete_node_logs(node, prefix)
def reset_environment_resp(cls, **kwargs): logger.info("RPC method reset_environment_resp received: %s", jsonutils.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True, lock_for_update=True) # Locking cluster objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True) if status == consts.TASK_STATUSES.ready: # restoring pending changes task.cluster.status = consts.CLUSTER_STATUSES.new objects.Cluster.add_pending_changes( task.cluster, consts.CLUSTER_CHANGES.attributes) objects.Cluster.add_pending_changes( task.cluster, consts.CLUSTER_CHANGES.networks) node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)] q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids) q_nodes = objects.NodeCollection.filter_by( q_nodes, cluster_id=task.cluster_id) q_nodes = objects.NodeCollection.order_by(q_nodes, 'id') # locking Nodes for update update_nodes = objects.NodeCollection.lock_for_update( q_nodes).all() for node in update_nodes: logs_utils.delete_node_logs(node) objects.Node.reset_to_discover(node) if ia_nodes: cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"environment resetting") message = (u"Environment '{0}' " u"was successfully reset".format(task.cluster.name or task.cluster_id)) notifier.notify("done", message, task.cluster_id) data = {'status': status, 'progress': progress, 'message': message} objects.Task.update(task, data) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def test_delete_node_logs(self): prefix = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, prefix) cluster = self.create_env([{'roles': ['controller']}]) node = cluster.nodes[0] log_paths = logs_utils.generate_log_paths_for_node(node, prefix) link = log_paths['links'][0] os.symlink(log_paths['old'], link) folder = log_paths['new'] os.mkdir(folder) file_ = log_paths['bak'] with open(file_, 'w') as f: f.write("RANDOMCONTENT") logs_utils.delete_node_logs(node, prefix) self.assertTrue( all(not os.path.lexists(path) for path in [link, folder, file_]))
def remove_nodes_resp(cls, **kwargs): logger.info( "RPC method remove_nodes_resp received: %s" % jsonutils.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] error_nodes = kwargs.get('error_nodes') or [] inaccessible_nodes = kwargs.get('inaccessible_nodes') or [] error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') if status in [consts.TASK_STATUSES.ready, consts.TASK_STATUSES.error]: progress = 100 # locking tasks on cluster task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True) objects.TaskCollection.lock_cluster_tasks(task.cluster_id) task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True, lock_for_update=True ) # locking cluster if task.cluster_id is not None: objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True ) # locking nodes all_nodes = itertools.chain(nodes, error_nodes, inaccessible_nodes) all_nodes_ids = [ node['id'] if 'id' in node else node['uid'] for node in all_nodes ] locked_nodes = objects.NodeCollection.filter_by_list( None, 'id', all_nodes_ids, order_by='id' ) objects.NodeCollection.lock_for_update(locked_nodes).all() def get_node_id(n): return n.get('id', int(n.get('uid'))) nodes_to_delete_ids = [get_node_id(n) for n in nodes] if(len(inaccessible_nodes) > 0): inaccessible_node_ids = [ get_node_id(n) for n in inaccessible_nodes] logger.warn(u'Nodes %s not answered by RPC, removing from db', inaccessible_nodes) nodes_to_delete_ids.extend(inaccessible_node_ids) for node in objects.NodeCollection.filter_by_id_list( None, nodes_to_delete_ids): logs_utils.delete_node_logs(node) objects.NodeCollection.delete_by_ids(nodes_to_delete_ids) for node in error_nodes: node_db = objects.Node.get_by_uid(node['uid']) if not node_db: logger.error( u"Failed to delete node '%s' marked as error from Astute:" " node doesn't exist", str(node) ) else: node_db.pending_deletion = False node_db.status = 'error' db().add(node_db) node['name'] = node_db.name db().flush() success_msg = u"No nodes were removed" err_msg = u"No errors occurred" if nodes: success_msg = u"Successfully removed {0} node(s)".format( len(nodes) ) notifier.notify("done", success_msg) if error_nodes: err_msg = u"Failed to remove {0} node(s): {1}".format( len(error_nodes), ', '.join( [n.get('name') or "ID: {0}".format(n['uid']) for n in error_nodes]) ) notifier.notify("error", err_msg) if not error_msg: error_msg = ". ".join([success_msg, err_msg]) data = { 'status': status, 'progress': progress, 'message': error_msg, } objects.Task.update(task, data) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def remove_nodes_resp(cls, **kwargs): logger.info( "RPC method remove_nodes_resp received: %s" % jsonutils.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] error_nodes = kwargs.get('error_nodes') or [] inaccessible_nodes = kwargs.get('inaccessible_nodes') or [] error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') if status in [consts.TASK_STATUSES.ready, consts.TASK_STATUSES.error]: progress = 100 # locking task task = objects.Task.get_by_uuid( task_uuid, fail_if_not_found=True, lock_for_update=True ) # locking cluster if task.cluster_id is not None: objects.Cluster.get_by_uid( task.cluster_id, fail_if_not_found=True, lock_for_update=True ) # locking nodes all_nodes = itertools.chain(nodes, error_nodes, inaccessible_nodes) all_nodes_ids = [ node['id'] if 'id' in node else node['uid'] for node in all_nodes ] locked_nodes = objects.NodeCollection.filter_by_list( None, 'id', all_nodes_ids, order_by='id' ) objects.NodeCollection.lock_for_update(locked_nodes).all() def get_node_id(n): return n.get('id', int(n.get('uid'))) nodes_to_delete_ids = [get_node_id(n) for n in nodes] if(len(inaccessible_nodes) > 0): inaccessible_node_ids = [ get_node_id(n) for n in inaccessible_nodes] logger.warn(u'Nodes %s not answered by RPC, removing from db', inaccessible_nodes) nodes_to_delete_ids.extend(inaccessible_node_ids) for node in objects.NodeCollection.filter_by_id_list( None, nodes_to_delete_ids): logs_utils.delete_node_logs(node) objects.NodeCollection.delete_by_ids(nodes_to_delete_ids) for node in error_nodes: node_db = objects.Node.get_by_uid(node['uid']) if not node_db: logger.error( u"Failed to delete node '%s' marked as error from Astute:" " node doesn't exist", str(node) ) else: node_db.pending_deletion = False node_db.status = 'error' db().add(node_db) node['name'] = node_db.name db().flush() success_msg = u"No nodes were removed" err_msg = u"No errors occurred" if nodes: success_msg = u"Successfully removed {0} node(s)".format( len(nodes) ) notifier.notify("done", success_msg) if error_nodes: err_msg = u"Failed to remove {0} node(s): {1}".format( len(error_nodes), ', '.join( [n.get('name') or "ID: {0}".format(n['uid']) for n in error_nodes]) ) notifier.notify("error", err_msg) if not error_msg: error_msg = ". ".join([success_msg, err_msg]) data = { 'status': status, 'progress': progress, 'message': error_msg, } objects.Task.update(task, data) cls._update_action_log_entry(status, task.name, task_uuid, nodes)