def destroy(self, service):
"""
Destroy a group of instances described by "service".
"""
logger.debug("Attempting to destroy: %s", service)
asg_name = AsgName(network=service.network.name, subnetwork=service.name)
self.asg.destroy_auto_scaling_group_instances(asg_name)
# Wait for instances to be gone. Need to do this before we can delete
# the actual ASG otherwise it will error.
def instance_list(service, state):
return [instance for subnetwork in service.subnetworks
for instance in subnetwork.instances
if instance.state == state]
asg = self.get(service.network, service.name)
retries = 0
while asg and instance_list(asg, "terminated"):
logger.info("Waiting for instance termination in asg: %s", asg)
asg = self.get(service.network, service.name)
retries = retries + 1
if retries > 60:
raise OperationTimedOut("Timed out waiting for ASG scale down")
time.sleep(float(10))
self.asg.destroy_auto_scaling_group(asg_name)
# Wait for ASG to be gone. Need to wait for this because it's a
# dependency of the launch configuration.
asg = self.get(service.network, service.name)
retries = 0
while asg:
logger.info("Waiting for asg deletion: %s", asg)
asg = self.get(service.network, service.name)
retries = retries + 1
if retries > 60:
raise OperationTimedOut("Timed out waiting for ASG deletion")
time.sleep(float(10))
vpc_id = service.network.network_id
lc_security_group = self.asg.get_launch_configuration_security_group(
service.network, service.name)
self.asg.destroy_launch_configuration(asg_name)
if lc_security_group:
self.security_groups.delete_referencing_rules(vpc_id,
lc_security_group)
self.security_groups.delete_with_retries(lc_security_group,
RETRY_COUNT, RETRY_DELAY)
else:
self.security_groups.delete_by_name(vpc_id, str(asg_name),
RETRY_COUNT, RETRY_DELAY)
self.subnetwork.destroy(service.network, service.name)
def attempt_delete_security_group(security_group_id):
try:
ec2.delete_security_group(GroupId=security_group_id)
return True
except ClientError as client_error:
logger.info("Recieved exception destroying security group: %s",
client_error)
if (client_error.response["Error"]["Code"] ==
"DependencyViolation"):
return False
raise client_error
def get_availability_zones(self):
"""
Returns the list of all availiablity zones in the current region.
"""
ec2 = self.driver.client("ec2")
if self.mock:
# NOTE: Moto does not have this function supported, so this has to be here to get the
# mock tests passing.
logger.info("Returning hard coded azs for mock AWS provider")
return ["us-east-1a", "us-east-1b", "us-east-1c"]
availability_zones = ec2.describe_availability_zones()
return [
az["ZoneName"] for az in availability_zones["AvailabilityZones"]
]
def destroy_launch_configuration(self, asg_name):
autoscaling = self.driver.client("autoscaling")
try:
autoscaling.delete_launch_configuration(
LaunchConfigurationName=str(asg_name))
except ClientError as client_error:
logger.info("Recieved exception destroying launch configuration: %s", client_error)
msg = "Launch configuration name not found - Launch configuration %s not found" % (
str(asg_name))
if (client_error.response["Error"]["Code"] == "ValidationError" and
client_error.response["Error"]["Message"] == msg):
logger.debug("Launch configuration: %s already deleted",
str(asg_name))
else:
raise client_error
def wait_until(state):
asg = self.get(network, service_name)
def instance_list(service, state):
return [instance for subnetwork in service.subnetworks
for instance in subnetwork.instances
if instance.state == state]
retries = 0
while len(instance_list(asg, state)) < instance_count:
logger.info("Waiting for instance creation in asg: %s", asg)
asg = self.get(network, service_name)
retries = retries + 1
if retries > RETRY_COUNT:
raise OperationTimedOut("Timed out waiting for ASG to be created")
time.sleep(RETRY_DELAY)
def create(self, subnetwork_name, subnet_cidr, availability_zone, dc_id,
retry_count, retry_delay):
"""
Provision a single subnet with a route table and the proper tags.
"""
ec2 = self.driver.client("ec2")
created_subnet = ec2.create_subnet(CidrBlock=subnet_cidr,
AvailabilityZone=availability_zone,
VpcId=dc_id)
subnet_id = created_subnet["Subnet"]["SubnetId"]
route_table = ec2.create_route_table(VpcId=dc_id)
route_table_id = route_table["RouteTable"]["RouteTableId"]
ec2.associate_route_table(RouteTableId=route_table_id,
SubnetId=subnet_id)
creation_retries = 0
while creation_retries < retry_count:
try:
ec2.create_tags(Resources=[subnet_id],
Tags=[{
"Key": "Name",
"Value": subnetwork_name
}])
subnets = ec2.describe_subnets(
Filters=[{
'Name': "vpc-id",
'Values': [dc_id]
}, {
'Name': "tag:Name",
'Values': [subnetwork_name]
}])
subnet_ids = [
subnet["SubnetId"] for subnet in subnets["Subnets"]
]
if subnet_id not in subnet_ids:
time.sleep(float(retry_delay))
else:
break
except ec2.exceptions.ClientError as client_error:
logger.info("Caught exception creating tags: %s", client_error)
time.sleep(float(retry_delay))
creation_retries = creation_retries + 1
if creation_retries >= retry_count:
raise OperationTimedOut("Cannot find created Subnet: %s" %
subnet_id)
return created_subnet["Subnet"]
def add(self, source, destination, port):
"""
Adds a route from "source" to "destination".
"""
logger.debug("Adding path from %s to %s", source, destination)
if self.has_access(source, destination, port):
logger.info("Service %s already has access to %s on port: %s",
source, destination, port)
return True
# Currently controlling egress in AWS is not supported. All egress is always allowed.
if not isinstance(destination, Service):
raise DisallowedOperationException(
"Destination must be a butter.types.networking.Service object")
dest_sg_id, _, _, src_ip_permissions = self._extract_service_info(
source, destination, port)
ec2 = self.driver.client("ec2")
ec2.authorize_security_group_ingress(GroupId=dest_sg_id,
IpPermissions=src_ip_permissions)
return Path(destination.network, source, destination, "tcp", port)
def delete_referencing_rules(self, vpc_id, security_group_id):
"""
Removes all rules referencing the given security group in the given
VPC, so it can be safely deleted.
"""
ec2 = self.driver.client("ec2")
logger.info("Deleting rules referencing %s in %s", security_group_id,
vpc_id)
security_groups = ec2.describe_security_groups(
Filters=[{
'Name': 'vpc-id',
'Values': [vpc_id]
}])
for security_group in security_groups["SecurityGroups"]:
logger.info("Checking security group: %s", security_group_id)
for rule in security_group["IpPermissions"]:
for uigp in rule["UserIdGroupPairs"]:
if "GroupId" not in uigp:
continue
if uigp["GroupId"] != security_group_id:
continue
rule_to_remove = {}
rule_to_remove["FromPort"] = rule["FromPort"]
rule_to_remove["ToPort"] = rule["ToPort"]
rule_to_remove["IpProtocol"] = rule["IpProtocol"]
rule_to_remove["UserIdGroupPairs"] = [{
"GroupId":
uigp["GroupId"]
}]
logger.info("Revoking rule: %s in security group %s",
rule_to_remove, security_group)
ec2.revoke_security_group_ingress(
GroupId=security_group["GroupId"],
IpPermissions=[rule_to_remove])
def delete_by_name(self, vpc_id, security_group_name, retries,
retry_delay):
ec2 = self.driver.client("ec2")
logger.info("Deleting security group %s in %s", security_group_name,
vpc_id)
security_groups = ec2.describe_security_groups(
Filters=[{
'Name': 'vpc-id',
'Values': [vpc_id]
}, {
'Name': 'group-name',
'Values': [security_group_name]
}])
if not security_groups["SecurityGroups"]:
return True
if len(security_groups["SecurityGroups"]) > 1:
raise BadEnvironmentStateException(
"Found multiple security groups with name %s, in vpc %s: %s" %
(security_group_name, vpc_id, security_groups))
security_group_id = security_groups["SecurityGroups"][0]["GroupId"]
return self.delete_with_retries(security_group_id, retries,
retry_delay)
def destroy(self, network, subnetwork_name):
"""
Destroy all networks represented by this object. Also destroys the
underlying VPC if it's empty.
Steps:
1. Discover the current VPC.
2. Destroy route tables.
2.a. Disassociate and delete route table.
2.b. Delete non referenced internet gateways.
3. Delete all subnets.
4. Wait until subnets are deleted.
"""
ec2 = self.driver.client("ec2")
subnet_ids = [
subnet_info.subnetwork_id
for subnet_info in self.get(network, subnetwork_name)
]
# 1. Discover the current VPC.
dc_id = network.network_id
# 2. Destroy route tables.
def delete_route_table(route_table):
# 2.a. Disassociate and delete route table.
associations = route_table["Associations"]
for association in associations:
ec2.disassociate_route_table(
AssociationId=association["RouteTableAssociationId"])
ec2.delete_route_table(RouteTableId=route_table["RouteTableId"])
# 2.b. Delete non referenced internet gateways.
routes = route_table["Routes"]
for route in routes:
if "GatewayId" in route and route["GatewayId"] != "local":
igw_id = route["GatewayId"]
if not self.internet_gateways.route_count(dc_id, igw_id):
ec2.detach_internet_gateway(InternetGatewayId=igw_id,
VpcId=dc_id)
ec2.delete_internet_gateway(InternetGatewayId=igw_id)
for subnet_id in subnet_ids:
subnet_filter = {
'Name': 'association.subnet-id',
'Values': [subnet_id]
}
route_tables = ec2.describe_route_tables(Filters=[subnet_filter])
if len(route_tables["RouteTables"]) > 1:
raise BadEnvironmentStateException(
"Expected to find at most one route table associated "
"with: %s, output: %s" % (subnet_id, route_tables))
if len(route_tables["RouteTables"]) == 1:
delete_route_table(route_tables["RouteTables"][0])
# 3. Delete all subnets.
for subnet_id in subnet_ids:
self.subnets.delete(subnet_id, RETRY_COUNT, RETRY_DELAY)
# 4. Wait until subnets are deleted.
remaining_subnets = ec2.describe_subnets(Filters=[{
'Name': 'vpc-id',
'Values': [dc_id]
}])
remaining_subnet_ids = [
subnet["SubnetId"] for subnet in remaining_subnets["Subnets"]
]
retries = 0
while (any(i in subnet_ids for i in remaining_subnet_ids)
and retries < 720):
logger.info("Found remaining subnets: %s", remaining_subnet_ids)
remaining_subnets = ec2.describe_subnets(Filters=[{
'Name': 'vpc-id',
'Values': [dc_id]
}])
remaining_subnet_ids = [
subnet["SubnetId"] for subnet in remaining_subnets["Subnets"]
]
retries = retries + 1
time.sleep(1)
def get(self, network, service_name):
"""
Discover a service in "network" named "service_name".
"""
logger.info("Discovering autoscaling group named %s in network: %s",
service_name, network)
def discover_asg(network_name, service_name):
autoscaling = self.driver.client("autoscaling")
logger.debug("Discovering auto scaling groups with name: %s",
service_name)
asg_name = AsgName(network=network_name, subnetwork=service_name)
asgs = autoscaling.describe_auto_scaling_groups(
AutoScalingGroupNames=[str(asg_name)])
logger.debug("Found asgs: %s", asgs)
if len(asgs["AutoScalingGroups"]) > 1:
raise BadEnvironmentStateException(
"Expected to find at most one auto scaling group "
"named: %s, output: %s" % (str(asg_name), asgs))
if not asgs["AutoScalingGroups"]:
return None
return asgs["AutoScalingGroups"][0]
def discover_instances(instance_ids):
ec2 = self.driver.client("ec2")
logger.debug("Discovering instances: %s", instance_ids)
instances = {"Reservations": []}
# Need this check because if we pass an empty list the API returns all instances
if instance_ids:
instances = ec2.describe_instances(InstanceIds=instance_ids)
return [instance
for reservation in instances["Reservations"]
for instance in reservation["Instances"]]
# 1. Get List Of Instances
discovery_retries = 0
discovery_complete = False
while discovery_retries < RETRY_COUNT:
try:
asg = discover_asg(network.name, service_name)
if not asg:
return None
instance_ids = [instance["InstanceId"] for instance in asg["Instances"]]
instances = discover_instances(instance_ids)
logger.debug("Discovered instances: %s", instances)
discovery_complete = True
except ClientError as client_error:
# There is a race between when I discover the autoscaling group
# itself and when I try to search for the instances inside it,
# so just retry if this happens.
logger.info("Recieved exception discovering instance: %s", client_error)
if client_error.response["Error"]["Code"] == "InvalidInstanceID.NotFound":
pass
else:
raise
if discovery_complete:
break
discovery_retries = discovery_retries + 1
logger.info("Instance discovery retry number: %s",
discovery_retries)
if discovery_retries >= RETRY_COUNT:
raise OperationTimedOut(
"Exceeded retries while discovering %s, in network %s" %
(service_name, network))
time.sleep(RETRY_DELAY)
# 2. Get List Of Subnets
subnetworks = self.subnetwork.get(network, service_name)
# NOTE: In moto instance objects do not include a "SubnetId" and the IP addresses are
# assigned randomly in the VPC, so for now just stripe instances across subnets.
if self.mock:
for instance, subnetwork, in zip(instances, itertools.cycle(subnetworks)):
subnetwork.instances.append(canonicalize_instance_info(instance))
return Service(network=network, name=service_name, subnetworks=subnetworks)
# 3. Group Services By Subnet
for subnetwork in subnetworks:
for instance in instances:
if "SubnetId" in instance and subnetwork.subnetwork_id == instance["SubnetId"]:
subnetwork.instances.append(canonicalize_instance_info(instance))
# 4. Profit!
return Service(network=network, name=service_name, subnetworks=subnetworks)
def destroy(self, network):
"""
Destroy a network given the provided network object.
"""
ec2 = self.driver.client("ec2")
# Check to see if we have any subnets, otherwise bail out
subnets = ec2.describe_subnets(Filters=[{
'Name': 'vpc-id',
'Values': [network.network_id]
}])
if subnets["Subnets"]:
message = "Found subnets in network, cannot delete: %s" % subnets
logger.error(message)
raise DisallowedOperationException(message)
# Delete internet gateway if it's no longer referenced
igw = ec2.describe_internet_gateways(
Filters=[{
'Name': 'attachment.vpc-id',
'Values': [network.network_id]
}])
igw_id = None
if len(igw["InternetGateways"]) == 1:
igw_id = igw["InternetGateways"][0]["InternetGatewayId"]
elif len(igw["InternetGateways"]) > 1:
raise Exception(
"Invalid response from describe_internet_gateways: %s" % igw)
if igw_id and not self.internet_gateways.route_count(
network.network_id, igw_id):
ec2.detach_internet_gateway(InternetGatewayId=igw_id,
VpcId=network.network_id)
ec2.delete_internet_gateway(InternetGatewayId=igw_id)
# Since we check above that there are no subnets, and therefore nothing
# deployed in this VPC, for now assume it is safe to delete.
security_groups = ec2.describe_security_groups(
Filters=[{
'Name': 'vpc-id',
'Values': [network.network_id]
}])
for security_group in security_groups["SecurityGroups"]:
if security_group["GroupName"] == "default":
continue
logger.info("Deleting security group: %s",
security_group["GroupName"])
ec2.delete_security_group(GroupId=security_group["GroupId"])
# Delete internet gateway, also safe because our subnets are gone.
igws = ec2.describe_internet_gateways(
Filters=[{
'Name': 'attachment.vpc-id',
'Values': [network.network_id]
}])
logger.info("Deleting internet gateways: %s", igws)
for igw in igws["InternetGateways"]:
logger.info("Deleting internet gateway: %s", igw)
igw_id = igw["InternetGatewayId"]
ec2.detach_internet_gateway(InternetGatewayId=igw_id,
VpcId=network.network_id)
ec2.delete_internet_gateway(InternetGatewayId=igw_id)
# Now, actually delete the VPC
try:
deletion_result = ec2.delete_vpc(VpcId=network.network_id)
except ec2.exceptions.ClientError as client_error:
if client_error.response['Error']['Code'] == 'DependencyViolation':
logger.info("Dependency violation deleting VPC: %s",
client_error)
raise client_error
return deletion_result