def discover_instances(instance_ids):
"""
Given instance IDs get the actual instance data for them.
"""
ec2 = self.driver.client("ec2")
logger.debug("Discovering instances: %s", instance_ids)
instances = {"Reservations": []}
# Need this check because if we pass an empty list the API returns all instances
if instance_ids:
instances = ec2.describe_instances(InstanceIds=instance_ids)
return [
instance for reservation in instances["Reservations"]
for instance in reservation["Instances"]
]
def list(self):
"""
List all images.
"""
ec2 = self.driver.client("ec2")
raw_images = ec2.describe_images(Owners=["self"])
logger.debug("Images: %s", raw_images)
images = []
for image in raw_images["Images"]:
images.append(
Image(image_id=image["ImageId"],
name=image["Name"],
created_at=image["CreationDate"]))
return images
def destroy_launch_configuration(self, asg_name):
autoscaling = self.driver.client("autoscaling")
try:
autoscaling.delete_launch_configuration(
LaunchConfigurationName=str(asg_name))
except ClientError as client_error:
logger.info("Recieved exception destroying launch configuration: %s", client_error)
msg = "Launch configuration name not found - Launch configuration %s not found" % (
str(asg_name))
if (client_error.response["Error"]["Code"] == "ValidationError" and
client_error.response["Error"]["Message"] == msg):
logger.debug("Launch configuration: %s already deleted",
str(asg_name))
else:
raise client_error
def get_launch_configuration_security_group(self, network_name,
subnetwork_name):
logger.debug("Getting security group for launch configuration %s, %s",
network_name, subnetwork_name)
asg_name = AsgName(network=network_name, subnetwork=subnetwork_name)
launch_configuration = self._describe_launch_configuration(asg_name)
if not launch_configuration:
return None
security_groups = launch_configuration["SecurityGroups"]
if len(security_groups) != 1:
raise BadEnvironmentStateException(
"Expected launch configuration "
"%s to have exactly one "
"security group: %s" % str(asg_name), launch_configuration)
return security_groups[0]
def wait_for_in_service(self, asg_name, instance_id):
logger.debug("Waiting for %s in %s to be in service", instance_id, asg_name)
autoscaling = self.driver.client("autoscaling")
asgs = autoscaling.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])
if len(asgs["AutoScalingGroups"]) != 1:
raise BadEnvironmentStateException(
"Found multiple asgs for name %s: %s" % (asg_name, asgs))
for instance in asgs["AutoScalingGroups"][0]["Instances"]:
if instance["InstanceId"] == instance_id:
if instance["LifecycleState"] == "InService":
return True
logger.debug("Still waiting for %s in %s to be in service", instance_id,
asg_name)
raise IncompleteOperationException("Still waiting for in service")
raise BadEnvironmentStateException(
"Could not find instance %s in asg: %s" % (instance_id, asgs))
def destroy_auto_scaling_group_instances(self, asg_name):
autoscaling = self.driver.client("autoscaling")
try:
autoscaling.update_auto_scaling_group(
AutoScalingGroupName=str(asg_name), MinSize=0,
DesiredCapacity=0)
except ClientError as client_error:
logger.debug("Recieved exception scaling autoscaling group: %s",
client_error)
msg = "AutoScalingGroup name not found - null"
if (client_error.response["Error"]["Code"] == "ValidationError" and
client_error.response["Error"]["Message"] == msg):
logger.debug("Autoscaling group: %s already deleted",
str(asg_name))
else:
raise client_error
def destroy_auto_scaling_group(self, asg_name):
autoscaling = self.driver.client("autoscaling")
try:
autoscaling.delete_auto_scaling_group(
AutoScalingGroupName=str(asg_name), ForceDelete=True)
except ClientError as client_error:
logger.debug("Recieved exception destroying autoscaling group: %s",
client_error)
msg = "AutoScalingGroup name not found - AutoScalingGroup '%s' not found" % (
str(asg_name))
if (client_error.response["Error"]["Code"] == "ValidationError"
and client_error.response["Error"]["Message"] == msg):
logger.debug("Autoscaling group: %s already deleted",
str(asg_name))
else:
raise client_error
def _discover_asg(self, network_name, service_name):
"""
Discover an autoscaling group given a network and service name.
"""
autoscaling = self.driver.client("autoscaling")
logger.debug("Discovering auto scaling groups with name: %s",
service_name)
asg_name = AsgName(network=network_name, subnetwork=service_name)
asgs = autoscaling.describe_auto_scaling_groups(
AutoScalingGroupNames=[str(asg_name)])
logger.debug("Found asgs: %s", asgs)
if len(asgs["AutoScalingGroups"]) > 1:
raise BadEnvironmentStateException(
"Expected to find at most one auto scaling group "
"named: %s, output: %s" % (str(asg_name), asgs))
if not asgs["AutoScalingGroups"]:
return None
return asgs["AutoScalingGroups"][0]
def has_access(self, source, destination, port):
"""
Return true if there is a route from "source" to "destination".
"""
ec2 = self.driver.client("ec2")
dest_sg_id, src_sg_id, _, src_ip_permissions = self._extract_service_info(
source, destination, port)
security_group = ec2.describe_security_groups(GroupIds=[dest_sg_id])
def extract_cidr_port(ip_permissions):
cidr_port_list = []
for ip_permission in ip_permissions:
if "IpRanges" in ip_permission:
for ip_range in ip_permission["IpRanges"]:
cidr_port_list.append({
"port": ip_permission["FromPort"],
"cidr": ip_range["CidrIp"]
})
return cidr_port_list
def sg_allowed(ip_permissions, sg_id, port):
for ip_permission in ip_permissions:
if (ip_permission["UserIdGroupPairs"]
and ip_permission["FromPort"] == port):
for pair in ip_permission["UserIdGroupPairs"]:
if "GroupId" in pair and pair["GroupId"] == sg_id:
return True
return False
ip_permissions = security_group["SecurityGroups"][0]["IpPermissions"]
logger.debug("ip_permissions: %s", ip_permissions)
if src_sg_id and sg_allowed(ip_permissions, src_sg_id, port):
return True
src_cidr_port_info = extract_cidr_port(src_ip_permissions)
cidr_port_info = extract_cidr_port(ip_permissions)
for cidr_port in cidr_port_info:
if cidr_port["port"] != port:
continue
for src_cidr_port in src_cidr_port_info:
if (ipaddress.IPv4Network(src_cidr_port["cidr"]).overlaps(
ipaddress.IPv4Network(cidr_port["cidr"]))):
return True
return False
def destroy(self, image):
"""
Destroy a image given the provided image object.
"""
ec2 = self.driver.client("ec2")
raw_images = ec2.describe_images(Owners=["self"])
logger.debug("Images: %s", raw_images)
snapshot_ids = []
for raw_image in raw_images["Images"]:
if raw_image["ImageId"] == image.image_id:
for bdm in raw_image["BlockDeviceMappings"]:
snapshot_id = bdm.get("Ebs", {}).get("SnapshotId")
if snapshot_id:
snapshot_ids.append(snapshot_id)
logger.info("Deregistering image: %s", image.image_id)
ec2.deregister_image(ImageId=image.image_id)
for snapshot_id in snapshot_ids:
logger.info("Deleting snapshot: %s", snapshot_id)
ec2.delete_snapshot(SnapshotId=snapshot_id)
def retry_if_timeout(exception):
"""
Checks if this exception is just because we haven't converged yet.
"""
return isinstance(exception, OperationTimedOut)
@retry(wait_fixed=RETRY_DELAY,
stop_max_attempt_number=RETRY_COUNT,
retry_on_exception=retry_if_timeout)
def wait_for_destroyed(image_name):
logger.info("Waiting for image: %s to be destroyed", image_name)
if self.get(image.name):
raise OperationTimedOut(
"Timed out waiting for image %s to be gone." % image_name)
logger.info("Success, did not find image: %s", image_name)
wait_for_destroyed(image.name)
return True
def add(self, source, destination, port):
"""
Adds a route from "source" to "destination".
"""
logger.debug("Adding path from %s to %s", source, destination)
if self.has_access(source, destination, port):
logger.info("Service %s already has access to %s on port: %s",
source, destination, port)
return True
# Currently controlling egress in AWS is not supported. All egress is always allowed.
if not isinstance(destination, Service):
raise DisallowedOperationException(
"Destination must be a cloudless.types.networking.Service object"
)
dest_sg_id, _, _, src_ip_permissions = self._extract_service_info(
source, destination, port)
ec2 = self.driver.client("ec2")
ec2.authorize_security_group_ingress(GroupId=dest_sg_id,
IpPermissions=src_ip_permissions)
return Path(destination.network, source, destination, "tcp", port)
def add_instances_to_subnetwork_list(network, service_name,
subnetworks):
"""
Add the instances for this service to the list of subnetworks. Returns the same
subnetwork list with the instances added.
"""
# 1. Get List Of Instances.
discovery_retries = 0
discovery_complete = False
while discovery_retries < RETRY_COUNT:
try:
asg = self._discover_asg(network.name, service_name)
if not asg:
return subnetworks
instance_ids = [
instance["InstanceId"] for instance in asg["Instances"]
]
instances = discover_instances(instance_ids)
logger.debug("Discovered instances: %s", instances)
discovery_complete = True
except ClientError as client_error:
# There is a race between when I discover the autoscaling group
# itself and when I try to search for the instances inside it,
# so just retry if this happens.
logger.debug("Recieved exception discovering instance: %s",
client_error)
if client_error.response["Error"][
"Code"] == "InvalidInstanceID.NotFound":
pass
else:
raise
if discovery_complete:
break
discovery_retries = discovery_retries + 1
logger.debug("Instance discovery retry number: %s",
discovery_retries)
if discovery_retries >= RETRY_COUNT:
raise OperationTimedOut(
"Exceeded retries while discovering %s, in network %s"
% (service_name, network))
time.sleep(RETRY_DELAY)
# 2. Add instances to subnets.
# NOTE: In moto instance objects do not include a "SubnetId" and the IP addresses are
# assigned randomly in the VPC, so for now just stripe instances across subnets.
if self.mock:
for instance, subnetwork, in zip(instances,
itertools.cycle(subnetworks)):
subnetwork.instances.append(
canonicalize_instance_info(instance))
return subnetworks
for subnetwork in subnetworks:
for instance in instances:
if "SubnetId" in instance and subnetwork.subnetwork_id == instance[
"SubnetId"]:
subnetwork.instances.append(
canonicalize_instance_info(instance))
return subnetworks
def get_blockdev_info():
raw_instance = get_instance(instances[0].instance_id)
logger.debug("Getting blockdev info from: %s", raw_instance)
if len(raw_instance["BlockDeviceMappings"]) != 1:
raise DisallowedOperationException(
"Currently only support saving instances with one blockdev, found %s"
% (raw_instance))
volume_id = raw_instance["BlockDeviceMappings"][0]["Ebs"][
"VolumeId"]
volumes = ec2.describe_volumes(VolumeIds=[volume_id])
if len(volumes["Volumes"]) != 1:
raise BadEnvironmentStateException(
"Found two volumes with the same id: %s" % volumes)
volume = volumes["Volumes"][0]
return {
"DeviceName":
raw_instance["BlockDeviceMappings"][0]["DeviceName"],
"Ebs": {
"Encrypted": volume["Encrypted"],
"DeleteOnTermination": True,
"VolumeSize": volume["Size"],
"VolumeType": volume["VolumeType"]
}
}
def wait_for_stopped(instance_id):
raw_instance = get_instance(instance_id)
logger.debug("Current state: %s", raw_instance)
if raw_instance["State"]["Name"] != "stopped":
raise OperationTimedOut(
"Timed out waiting for instance: %s to stop" % instance_id)
def list(self):
"""
List all paths and return a dictionary structure representing a graph.
"""
ec2 = self.driver.client("ec2")
sg_to_service = {}
for service in self.service.list():
sg_id = self.asg.get_launch_configuration_security_group(
service.network.name, service.name)
if sg_id in sg_to_service:
sg_to_service[sg_id].append(service)
else:
sg_to_service[sg_id] = [service]
security_groups = ec2.describe_security_groups()
def make_path(destination, source, rule):
return Path(destination.network, source, destination,
rule["IpProtocol"], rule.get("FromPort", "N/A"))
def get_cidr_paths(destination, ip_permissions):
subnets = []
for ip_range in ip_permissions["IpRanges"]:
subnets.append(
Subnetwork(subnetwork_id=None,
name=None,
cidr_block=ip_range["CidrIp"],
region=None,
availability_zone=None,
instances=[]))
# We treat an explicit CIDR block as a special case of a service with no name.
paths = []
if subnets:
source = Service(network=None, name=None, subnetworks=subnets)
paths.append(make_path(destination, source, ip_permissions))
return paths
def get_sg_paths(destination, ip_permissions):
paths = []
for group in ip_permissions["UserIdGroupPairs"]:
services = sg_to_service[group["GroupId"]]
for service in services:
paths.append(
make_path(destination, service, ip_permissions))
return paths
paths = []
for security_group in security_groups["SecurityGroups"]:
if security_group["GroupId"] not in sg_to_service:
logger.debug(
"Security group %s is apparently not attached to a service. Skipping",
security_group["GroupId"])
continue
services = sg_to_service[security_group["GroupId"]]
for service in services:
for ip_permissions in security_group["IpPermissions"]:
logger.debug("ip_permissions: %s", ip_permissions)
paths.extend(get_cidr_paths(service, ip_permissions))
paths.extend(get_sg_paths(service, ip_permissions))
return paths
def destroy(self, service):
"""
Destroy a group of instances described by "service".
"""
logger.debug("Attempting to destroy: %s", service)
asg_name = AsgName(network=service.network.name,
subnetwork=service.name)
asg = self._discover_asg(service.network.name, service.name)
if asg:
self.asg.destroy_auto_scaling_group_instances(asg_name)
# Wait for instances to be gone. Need to do this before we can delete
# the actual ASG otherwise it will error.
def instance_list(service, state):
return [
instance for subnetwork in service.subnetworks
for instance in subnetwork.instances if instance.state != state
]
service = self.get(service.network, service.name)
logger.debug("Found service: %s", service)
retries = 0
while service and instance_list(service, "terminated"):
logger.info(
"Waiting for instance termination in service %s. %s still terminating",
service.name, len(instance_list(service, "terminated")))
logger.debug("Waiting for instance termination in asg: %s",
service)
service = self.get(service.network, service.name)
retries = retries + 1
if retries > RETRY_COUNT:
raise OperationTimedOut("Timed out waiting for ASG scale down")
time.sleep(RETRY_DELAY)
logger.info("Success! All instances terminated.")
asg = self._discover_asg(service.network.name, service.name)
if asg:
self.asg.destroy_auto_scaling_group(asg_name)
# Wait for ASG to be gone. Need to wait for this because it's a dependency of the launch
# configuration.
asg = self._discover_asg(service.network.name, service.name)
retries = 0
while asg:
logger.debug("Waiting for asg deletion: %s", asg)
asg = self._discover_asg(service.network.name, service.name)
retries = retries + 1
if retries > RETRY_COUNT:
raise OperationTimedOut("Timed out waiting for ASG deletion")
time.sleep(RETRY_DELAY)
vpc_id = service.network.network_id
lc_security_group = self.asg.get_launch_configuration_security_group(
service.network.name, service.name)
self.asg.destroy_launch_configuration(asg_name)
if lc_security_group:
logger.debug("Deleting referencing rules of sg: %s",
lc_security_group)
self.security_groups.delete_referencing_rules(
vpc_id, lc_security_group)
logger.debug("Attempting to delete sg: %s", lc_security_group)
self.security_groups.delete_with_retries(lc_security_group,
RETRY_COUNT, RETRY_DELAY)
else:
logger.debug("Attempting to delete sg by name: %s", str(asg_name))
self.security_groups.delete_by_name(vpc_id, str(asg_name),
RETRY_COUNT, RETRY_DELAY)
self.subnetwork.destroy(service.network, service.name)
def get(self, network, service_name):
"""
Discover a service in "network" named "service_name".
"""
logger.debug("Discovering autoscaling group named %s in network: %s",
service_name, network)
def discover_instances(instance_ids):
"""
Given instance IDs get the actual instance data for them.
"""
ec2 = self.driver.client("ec2")
logger.debug("Discovering instances: %s", instance_ids)
instances = {"Reservations": []}
# Need this check because if we pass an empty list the API returns all instances
if instance_ids:
instances = ec2.describe_instances(InstanceIds=instance_ids)
return [
instance for reservation in instances["Reservations"]
for instance in reservation["Instances"]
]
def add_instances_to_subnetwork_list(network, service_name,
subnetworks):
"""
Add the instances for this service to the list of subnetworks. Returns the same
subnetwork list with the instances added.
"""
# 1. Get List Of Instances.
discovery_retries = 0
discovery_complete = False
while discovery_retries < RETRY_COUNT:
try:
asg = self._discover_asg(network.name, service_name)
if not asg:
return subnetworks
instance_ids = [
instance["InstanceId"] for instance in asg["Instances"]
]
instances = discover_instances(instance_ids)
logger.debug("Discovered instances: %s", instances)
discovery_complete = True
except ClientError as client_error:
# There is a race between when I discover the autoscaling group
# itself and when I try to search for the instances inside it,
# so just retry if this happens.
logger.debug("Recieved exception discovering instance: %s",
client_error)
if client_error.response["Error"][
"Code"] == "InvalidInstanceID.NotFound":
pass
else:
raise
if discovery_complete:
break
discovery_retries = discovery_retries + 1
logger.debug("Instance discovery retry number: %s",
discovery_retries)
if discovery_retries >= RETRY_COUNT:
raise OperationTimedOut(
"Exceeded retries while discovering %s, in network %s"
% (service_name, network))
time.sleep(RETRY_DELAY)
# 2. Add instances to subnets.
# NOTE: In moto instance objects do not include a "SubnetId" and the IP addresses are
# assigned randomly in the VPC, so for now just stripe instances across subnets.
if self.mock:
for instance, subnetwork, in zip(instances,
itertools.cycle(subnetworks)):
subnetwork.instances.append(
canonicalize_instance_info(instance))
return subnetworks
for subnetwork in subnetworks:
for instance in instances:
if "SubnetId" in instance and subnetwork.subnetwork_id == instance[
"SubnetId"]:
subnetwork.instances.append(
canonicalize_instance_info(instance))
return subnetworks
# 1. Get List Of subnetworks. The service exists iff this exists.
subnetworks = self.subnetwork.get(network, service_name)
if not subnetworks:
return None
# 2. Add instances to subnetworks.
subnetworks = add_instances_to_subnetwork_list(network, service_name,
subnetworks)
# 3. Profit!
return Service(network=network,
name=service_name,
subnetworks=subnetworks)
