def test_terminate_normal_instance(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_id = "i-1234567890abcdef0"
client.describe_instances.return_value = {
'Reservations': [{'Instances': [{
'InstanceId': inst_id, 'InstanceLifecycle': 'normal'}]}]}
terminate_instance(inst_id)
client.terminate_instances.assert_called_with(InstanceIds=[inst_id])
def test_terminate_random_az_instance(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_id = "i-987654321fedcba"
client.describe_instances.return_value = {
'Reservations': [{'Instances': [{
'InstanceId': inst_id, 'InstanceLifecycle': 'normal'}]}]}
terminate_instance(az="us-west-1")
client.terminate_instances.assert_called_with(InstanceIds=[inst_id])
def test_terminate_instance_az_no_instances(aws_client):
client = MagicMock()
aws_client.return_value = client
az = 'us-west-2'
client.describe_instances.return_value = {
'Reservations': [{'Instances': []}]}
filters = [{'Name': 'availability-zone', 'Values': ['us-west-2']}]
with pytest.raises(FailedActivity) as x:
terminate_instance(az=az)
assert "No instances found matching filters: %s" % filters in str(x.value)
def test_terminate_spot_instance(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_id = "i-1234567890abcdef0"
spot_request_id = 'sir-abcdef01'
client.describe_instances.return_value = {
'Reservations': [{'Instances': [{
'InstanceId': inst_id, 'InstanceLifecycle': 'spot',
'SpotInstanceRequestId': spot_request_id}]}]}
terminate_instance(inst_id)
client.cancel_spot_instance_requests.assert_called_with(
SpotInstanceRequestIds=[spot_request_id])
client.terminate_instances.assert_called_with(InstanceIds=[inst_id])
def test_terminate_instance_az_no_instances(aws_client):
client = MagicMock()
aws_client.return_value = client
az = 'us-west-2'
client.describe_instances.return_value = {
'Reservations': [{
'Instances': []
}]
}
with pytest.raises(FailedActivity) as x:
terminate_instance(az=az)
assert 'No instances found matching filters: %s' % az
def test_terminate_instance_az_no_instances(aws_client):
client = MagicMock()
aws_client.return_value = client
az = "us-west-2"
client.describe_instances.return_value = {
"Reservations": [{
"Instances": []
}]
}
filters = [{"Name": "availability-zone", "Values": ["us-west-2"]}]
with pytest.raises(FailedActivity) as x:
terminate_instance(az=az)
assert "No instances found matching filters: %s" % filters in str(x.value)
def test_terminate_normal_instance(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_id = "i-1234567890abcdef0"
client.describe_instances.return_value = {
"Reservations": [{
"Instances": [{
"InstanceId": inst_id,
"InstanceLifecycle": "normal"
}]
}]
}
terminate_instance(inst_id)
client.terminate_instances.assert_called_with(InstanceIds=[inst_id])
def test_terminate_spot_instance(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_id = "i-1234567890abcdef0"
spot_request_id = "sir-abcdef01"
client.describe_instances.return_value = {
"Reservations": [{
"Instances": [{
"InstanceId": inst_id,
"InstanceLifecycle": "spot",
"SpotInstanceRequestId": spot_request_id,
}]
}]
}
terminate_instance(inst_id)
client.cancel_spot_instance_requests.assert_called_with(
SpotInstanceRequestIds=[spot_request_id])
client.terminate_instances.assert_called_with(InstanceIds=[inst_id])
def test_terminate_instance_by_filters(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_1_id = "i-987654321fedcba"
client.describe_instances.return_value = {
'Reservations': [{
'Instances': [{
'InstanceId': inst_1_id,
'InstanceLifecycle': 'normal'
}]
}]
}
filters = [
{
'Name': 'instance-state-name',
'Values': ['running']
},
{
'Name': 'tag-value',
'Values': ['chaos-cluster']
},
{
'Name': 'tag-key',
'Values': ['kubernetes.io/cluster/chaos-cluster']
},
{
'Name': 'tag-value',
'Values': ['owned']
},
{
'Name': 'tag-key',
'Values': ['eksctl.cluster.k8s.io/v1alpha1/cluster-name']
},
]
terminate_instance(filters=filters, az='us-west-2')
called_filters = deepcopy(filters)
called_filters.append({
'Name': 'availability-zone',
'Values': ['us-west-2']
})
client.describe_instances.assert_called_with(Filters=called_filters)
client.terminate_instances.assert_called_with(InstanceIds=[inst_1_id])
def test_terminate_instance_by_filters(aws_client):
client = MagicMock()
aws_client.return_value = client
inst_1_id = "i-987654321fedcba"
client.describe_instances.return_value = {
"Reservations": [{
"Instances": [{
"InstanceId": inst_1_id,
"InstanceLifecycle": "normal"
}]
}]
}
filters = [
{
"Name": "instance-state-name",
"Values": ["running"]
},
{
"Name": "tag-value",
"Values": ["chaos-cluster"]
},
{
"Name": "tag-key",
"Values": ["kubernetes.io/cluster/chaos-cluster"]
},
{
"Name": "tag-value",
"Values": ["owned"]
},
{
"Name": "tag-key",
"Values": ["eksctl.cluster.k8s.io/v1alpha1/cluster-name"]
},
]
terminate_instance(filters=filters, az="us-west-2")
called_filters = deepcopy(filters)
called_filters.append({
"Name": "availability-zone",
"Values": ["us-west-2"]
})
client.describe_instances.assert_called_with(Filters=called_filters)
client.terminate_instances.assert_called_with(InstanceIds=[inst_1_id])
def terminate_random_nodes(cluster_name: str,
aws_region: str,
node_count: int,
termination_timeout: int = 60,
secrets: Secrets = None):
"""
Terminate random EKS nodes.
The cluster_name must be specified so that a node instance (or multiple)
can be terminated. termination_timeout controls how long a termination
should take and is set to 60 by default; if the instance cannot be
terminated within this timeout, the action will fail.
"""
ec2_client = aws_client("ec2", {"aws_region": aws_region}, secrets)
ec2_describe_response = ec2_client.describe_instances(
Filters=[{
'Name': 'instance-state-name',
'Values': ['running']
}, {
'Name': 'network-interface.group-name',
'Values': ["{}-workers".format(cluster_name)]
}])
cluster_instances = []
for reservation in ec2_describe_response['Reservations']:
for instance in reservation['Instances']:
cluster_instances.append(instance['InstanceId'])
instances_to_terminate = random.sample(cluster_instances, node_count)
for instanceId in instances_to_terminate:
logger.info("Terminating {} instance".format(instanceId))
terminate_instance(instance_id=instanceId,
configuration={"aws_region": aws_region})
timeout = datetime.datetime.now() + datetime. \
timedelta(0, termination_timeout)
_wait_for(
timeout,
5, # check every 5 seconds if instance was successfully terminated
_instance_to_reach_terminated_state,
"waiting for the instance to reach a terminated state",
ec2_client,
instanceId)
def test_terminate_instance_no_values():
with pytest.raises(FailedActivity) as x:
terminate_instance()
assert 'To terminate an EC2, you must specify the instance-id, ' \
'an Availability Zone, or provide a set of filters' in str(x.value)
