def validate_cluster_health(asg_name, new_desired_asg_capacity,
desired_k8s_node_count):
cluster_healthy = False
# check if asg has enough nodes first before checking instance health
if is_asg_scaled(asg_name, new_desired_asg_capacity):
# if asg is healthy start draining and terminating instances
if is_asg_healthy(asg_name):
# check if k8s nodes are all online
if k8s_nodes_count(desired_k8s_node_count):
# check k8s nodes are healthy
if k8s_nodes_ready():
logger.info(
'Cluster validation passed. Proceeding with node draining and termination...'
)
cluster_healthy = True
else:
logger.info(
'Validation failed for cluster. Expected node count reached but nodes are not healthy.'
)
else:
nodes = get_k8s_nodes()
logger.info('Current k8s node count is {}'.format(len(nodes)))
logger.info(
'Validation failed for cluster. Current node count {} Expected node count {}.'
.format(len(nodes), desired_k8s_node_count))
else:
logger.info('Validation failed for asg {}.'
'Instances not healthy'.format(asg_name))
else:
logger.info('Validation failed for asg {}.'
'Not enough instances online'.format(asg_name))
return cluster_healthy
def test_is_asg_healthy_fail(self):
with patch('lib.aws.client.describe_auto_scaling_groups'
) as describe_auto_scaling_groups_mock:
describe_auto_scaling_groups_mock.return_value = self.aws_response_mock_unhealthy
result = is_asg_healthy('mock-asg', 2, 1)
self.assertFalse(result)
def test_is_asg_healthy(self):
result = is_asg_healthy('mock-asg', 2, 1)
self.assertTrue(result)
