def reset_environment_resp(cls, **kwargs):
logger.info("RPC method reset_environment_resp received: %s",
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(task_uuid,
fail_if_not_found=True,
lock_for_update=True)
# Locking cluster
objects.Cluster.get_by_uid(task.cluster_id,
fail_if_not_found=True,
lock_for_update=True)
if status == consts.TASK_STATUSES.ready:
# restoring pending changes
task.cluster.status = consts.CLUSTER_STATUSES.new
objects.Cluster.add_pending_changes(
task.cluster, consts.CLUSTER_CHANGES.attributes)
objects.Cluster.add_pending_changes(
task.cluster, consts.CLUSTER_CHANGES.networks)
node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)]
q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids)
q_nodes = objects.NodeCollection.filter_by(
q_nodes, cluster_id=task.cluster_id)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
# locking Nodes for update
update_nodes = objects.NodeCollection.lock_for_update(
q_nodes).all()
for node in update_nodes:
logs_utils.delete_node_logs(node)
objects.Node.reset_to_discover(node)
if ia_nodes:
cls._notify_inaccessible(task.cluster_id,
[n["uid"] for n in ia_nodes],
u"environment resetting")
message = (u"Environment '{0}' "
u"was successfully reset".format(task.cluster.name
or task.cluster_id))
notifier.notify("done", message, task.cluster_id)
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def test_delete_node_no_existing_logs(self, _):
"""Only checks whether errors are passing silently.
That's why there's no assertions, just expecting no errors.
"""
prefix = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, prefix)
cluster = self.create_env([{'roles': ['controller']}])
node = cluster.nodes[0]
logs_utils.delete_node_logs(node, prefix)
def test_delete_node_no_existing_logs(self, _):
"""Only checks whether errors are passing silently.
That's why there's no assertions, just expecting no errors.
"""
prefix = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, prefix)
cluster = self.create_env([{'roles': ['controller']}])
node = cluster.nodes[0]
logs_utils.delete_node_logs(node, prefix)
def reset_environment_resp(cls, **kwargs):
logger.info("RPC method reset_environment_resp received: %s",
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = objects.Task.get_by_uuid(
task_uuid, fail_if_not_found=True, lock_for_update=True)
# Locking cluster
objects.Cluster.get_by_uid(
task.cluster_id, fail_if_not_found=True, lock_for_update=True)
if status == consts.TASK_STATUSES.ready:
# restoring pending changes
task.cluster.status = consts.CLUSTER_STATUSES.new
objects.Cluster.add_pending_changes(
task.cluster, consts.CLUSTER_CHANGES.attributes)
objects.Cluster.add_pending_changes(
task.cluster, consts.CLUSTER_CHANGES.networks)
node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)]
q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids)
q_nodes = objects.NodeCollection.filter_by(
q_nodes, cluster_id=task.cluster_id)
q_nodes = objects.NodeCollection.order_by(q_nodes, 'id')
# locking Nodes for update
update_nodes = objects.NodeCollection.lock_for_update(
q_nodes).all()
for node in update_nodes:
logs_utils.delete_node_logs(node)
objects.Node.reset_to_discover(node)
if ia_nodes:
cls._notify_inaccessible(task.cluster_id,
[n["uid"] for n in ia_nodes],
u"environment resetting")
message = (u"Environment '{0}' "
u"was successfully reset".format(task.cluster.name
or task.cluster_id))
notifier.notify("done", message, task.cluster_id)
data = {'status': status, 'progress': progress, 'message': message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def test_delete_node_logs(self):
prefix = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, prefix)
cluster = self.create_env([{'roles': ['controller']}])
node = cluster.nodes[0]
log_paths = logs_utils.generate_log_paths_for_node(node, prefix)
link = log_paths['links'][0]
os.symlink(log_paths['old'], link)
folder = log_paths['new']
os.mkdir(folder)
file_ = log_paths['bak']
with open(file_, 'w') as f:
f.write("RANDOMCONTENT")
logs_utils.delete_node_logs(node, prefix)
self.assertTrue(
all(not os.path.lexists(path) for path in [link, folder, file_]))
def remove_nodes_resp(cls, **kwargs):
logger.info(
"RPC method remove_nodes_resp received: %s" %
jsonutils.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
error_nodes = kwargs.get('error_nodes') or []
inaccessible_nodes = kwargs.get('inaccessible_nodes') or []
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
if status in [consts.TASK_STATUSES.ready, consts.TASK_STATUSES.error]:
progress = 100
# locking tasks on cluster
task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
objects.TaskCollection.lock_cluster_tasks(task.cluster_id)
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True,
lock_for_update=True
)
# locking cluster
if task.cluster_id is not None:
objects.Cluster.get_by_uid(
task.cluster_id,
fail_if_not_found=True,
lock_for_update=True
)
# locking nodes
all_nodes = itertools.chain(nodes, error_nodes, inaccessible_nodes)
all_nodes_ids = [
node['id'] if 'id' in node else node['uid']
for node in all_nodes
]
locked_nodes = objects.NodeCollection.filter_by_list(
None,
'id',
all_nodes_ids,
order_by='id'
)
objects.NodeCollection.lock_for_update(locked_nodes).all()
def get_node_id(n):
return n.get('id', int(n.get('uid')))
nodes_to_delete_ids = [get_node_id(n) for n in nodes]
if(len(inaccessible_nodes) > 0):
inaccessible_node_ids = [
get_node_id(n) for n in inaccessible_nodes]
logger.warn(u'Nodes %s not answered by RPC, removing from db',
inaccessible_nodes)
nodes_to_delete_ids.extend(inaccessible_node_ids)
for node in objects.NodeCollection.filter_by_id_list(
None, nodes_to_delete_ids):
logs_utils.delete_node_logs(node)
objects.NodeCollection.delete_by_ids(nodes_to_delete_ids)
for node in error_nodes:
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.error(
u"Failed to delete node '%s' marked as error from Astute:"
" node doesn't exist", str(node)
)
else:
node_db.pending_deletion = False
node_db.status = 'error'
db().add(node_db)
node['name'] = node_db.name
db().flush()
success_msg = u"No nodes were removed"
err_msg = u"No errors occurred"
if nodes:
success_msg = u"Successfully removed {0} node(s)".format(
len(nodes)
)
notifier.notify("done", success_msg)
if error_nodes:
err_msg = u"Failed to remove {0} node(s): {1}".format(
len(error_nodes),
', '.join(
[n.get('name') or "ID: {0}".format(n['uid'])
for n in error_nodes])
)
notifier.notify("error", err_msg)
if not error_msg:
error_msg = ". ".join([success_msg, err_msg])
data = {
'status': status,
'progress': progress,
'message': error_msg,
}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def remove_nodes_resp(cls, **kwargs):
logger.info(
"RPC method remove_nodes_resp received: %s" %
jsonutils.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
error_nodes = kwargs.get('error_nodes') or []
inaccessible_nodes = kwargs.get('inaccessible_nodes') or []
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
if status in [consts.TASK_STATUSES.ready, consts.TASK_STATUSES.error]:
progress = 100
# locking task
task = objects.Task.get_by_uuid(
task_uuid,
fail_if_not_found=True,
lock_for_update=True
)
# locking cluster
if task.cluster_id is not None:
objects.Cluster.get_by_uid(
task.cluster_id,
fail_if_not_found=True,
lock_for_update=True
)
# locking nodes
all_nodes = itertools.chain(nodes, error_nodes, inaccessible_nodes)
all_nodes_ids = [
node['id'] if 'id' in node else node['uid']
for node in all_nodes
]
locked_nodes = objects.NodeCollection.filter_by_list(
None,
'id',
all_nodes_ids,
order_by='id'
)
objects.NodeCollection.lock_for_update(locked_nodes).all()
def get_node_id(n):
return n.get('id', int(n.get('uid')))
nodes_to_delete_ids = [get_node_id(n) for n in nodes]
if(len(inaccessible_nodes) > 0):
inaccessible_node_ids = [
get_node_id(n) for n in inaccessible_nodes]
logger.warn(u'Nodes %s not answered by RPC, removing from db',
inaccessible_nodes)
nodes_to_delete_ids.extend(inaccessible_node_ids)
for node in objects.NodeCollection.filter_by_id_list(
None, nodes_to_delete_ids):
logs_utils.delete_node_logs(node)
objects.NodeCollection.delete_by_ids(nodes_to_delete_ids)
for node in error_nodes:
node_db = objects.Node.get_by_uid(node['uid'])
if not node_db:
logger.error(
u"Failed to delete node '%s' marked as error from Astute:"
" node doesn't exist", str(node)
)
else:
node_db.pending_deletion = False
node_db.status = 'error'
db().add(node_db)
node['name'] = node_db.name
db().flush()
success_msg = u"No nodes were removed"
err_msg = u"No errors occurred"
if nodes:
success_msg = u"Successfully removed {0} node(s)".format(
len(nodes)
)
notifier.notify("done", success_msg)
if error_nodes:
err_msg = u"Failed to remove {0} node(s): {1}".format(
len(error_nodes),
', '.join(
[n.get('name') or "ID: {0}".format(n['uid'])
for n in error_nodes])
)
notifier.notify("error", err_msg)
if not error_msg:
error_msg = ". ".join([success_msg, err_msg])
data = {
'status': status,
'progress': progress,
'message': error_msg,
}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
