def validate_cluster_health(asg_name, new_desired_asg_capacity,
desired_k8s_node_count):
cluster_healthy = False
# check if asg has enough nodes first before checking instance health
if is_asg_scaled(asg_name, new_desired_asg_capacity):
# if asg is healthy start draining and terminating instances
if is_asg_healthy(asg_name):
# check if k8s nodes are all online
if k8s_nodes_count(desired_k8s_node_count):
# check k8s nodes are healthy
if k8s_nodes_ready():
logger.info(
'Cluster validation passed. Proceeding with node draining and termination...'
)
cluster_healthy = True
else:
logger.info(
'Validation failed for cluster. Expected node count reached but nodes are not healthy.'
)
else:
nodes = get_k8s_nodes()
logger.info('Current k8s node count is {}'.format(len(nodes)))
logger.info(
'Validation failed for cluster. Current node count {} Expected node count {}.'
.format(len(nodes), desired_k8s_node_count))
else:
logger.info('Validation failed for asg {}.'
'Instances not healthy'.format(asg_name))
else:
logger.info('Validation failed for asg {}.'
'Not enough instances online'.format(asg_name))
return cluster_healthy
def test_k8s_nodes_ready_fail(self):
with patch('lib.k8s.get_k8s_nodes') as get_k8s_nodes_mock:
box = Box(self.k8s_response_mock_unhealthy, ordered_box=True)
get_k8s_nodes_mock.return_value = box['items']
self.assertFalse(k8s_nodes_ready(2, 1), False)
def test_k8s_nodes_ready(self):
with patch('lib.k8s.get_k8s_nodes') as get_k8s_nodes_mock:
box = Box(self.k8s_response_mock, ordered_box=True)
get_k8s_nodes_mock.return_value = box['items']
self.assertTrue(k8s_nodes_ready(2, 1), True)