def discover_instances(instance_ids): """ Given instance IDs get the actual instance data for them. """ ec2 = self.driver.client("ec2") logger.debug("Discovering instances: %s", instance_ids) instances = {"Reservations": []} # Need this check because if we pass an empty list the API returns all instances if instance_ids: instances = ec2.describe_instances(InstanceIds=instance_ids) return [ instance for reservation in instances["Reservations"] for instance in reservation["Instances"] ]
def list(self): """ List all images. """ ec2 = self.driver.client("ec2") raw_images = ec2.describe_images(Owners=["self"]) logger.debug("Images: %s", raw_images) images = [] for image in raw_images["Images"]: images.append( Image(image_id=image["ImageId"], name=image["Name"], created_at=image["CreationDate"])) return images
def destroy_launch_configuration(self, asg_name): autoscaling = self.driver.client("autoscaling") try: autoscaling.delete_launch_configuration( LaunchConfigurationName=str(asg_name)) except ClientError as client_error: logger.info("Recieved exception destroying launch configuration: %s", client_error) msg = "Launch configuration name not found - Launch configuration %s not found" % ( str(asg_name)) if (client_error.response["Error"]["Code"] == "ValidationError" and client_error.response["Error"]["Message"] == msg): logger.debug("Launch configuration: %s already deleted", str(asg_name)) else: raise client_error
def get_launch_configuration_security_group(self, network_name, subnetwork_name): logger.debug("Getting security group for launch configuration %s, %s", network_name, subnetwork_name) asg_name = AsgName(network=network_name, subnetwork=subnetwork_name) launch_configuration = self._describe_launch_configuration(asg_name) if not launch_configuration: return None security_groups = launch_configuration["SecurityGroups"] if len(security_groups) != 1: raise BadEnvironmentStateException( "Expected launch configuration " "%s to have exactly one " "security group: %s" % str(asg_name), launch_configuration) return security_groups[0]
def wait_for_in_service(self, asg_name, instance_id): logger.debug("Waiting for %s in %s to be in service", instance_id, asg_name) autoscaling = self.driver.client("autoscaling") asgs = autoscaling.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) if len(asgs["AutoScalingGroups"]) != 1: raise BadEnvironmentStateException( "Found multiple asgs for name %s: %s" % (asg_name, asgs)) for instance in asgs["AutoScalingGroups"][0]["Instances"]: if instance["InstanceId"] == instance_id: if instance["LifecycleState"] == "InService": return True logger.debug("Still waiting for %s in %s to be in service", instance_id, asg_name) raise IncompleteOperationException("Still waiting for in service") raise BadEnvironmentStateException( "Could not find instance %s in asg: %s" % (instance_id, asgs))
def destroy_auto_scaling_group_instances(self, asg_name): autoscaling = self.driver.client("autoscaling") try: autoscaling.update_auto_scaling_group( AutoScalingGroupName=str(asg_name), MinSize=0, DesiredCapacity=0) except ClientError as client_error: logger.debug("Recieved exception scaling autoscaling group: %s", client_error) msg = "AutoScalingGroup name not found - null" if (client_error.response["Error"]["Code"] == "ValidationError" and client_error.response["Error"]["Message"] == msg): logger.debug("Autoscaling group: %s already deleted", str(asg_name)) else: raise client_error
def destroy_auto_scaling_group(self, asg_name): autoscaling = self.driver.client("autoscaling") try: autoscaling.delete_auto_scaling_group( AutoScalingGroupName=str(asg_name), ForceDelete=True) except ClientError as client_error: logger.debug("Recieved exception destroying autoscaling group: %s", client_error) msg = "AutoScalingGroup name not found - AutoScalingGroup '%s' not found" % ( str(asg_name)) if (client_error.response["Error"]["Code"] == "ValidationError" and client_error.response["Error"]["Message"] == msg): logger.debug("Autoscaling group: %s already deleted", str(asg_name)) else: raise client_error
def _discover_asg(self, network_name, service_name): """ Discover an autoscaling group given a network and service name. """ autoscaling = self.driver.client("autoscaling") logger.debug("Discovering auto scaling groups with name: %s", service_name) asg_name = AsgName(network=network_name, subnetwork=service_name) asgs = autoscaling.describe_auto_scaling_groups( AutoScalingGroupNames=[str(asg_name)]) logger.debug("Found asgs: %s", asgs) if len(asgs["AutoScalingGroups"]) > 1: raise BadEnvironmentStateException( "Expected to find at most one auto scaling group " "named: %s, output: %s" % (str(asg_name), asgs)) if not asgs["AutoScalingGroups"]: return None return asgs["AutoScalingGroups"][0]
def has_access(self, source, destination, port): """ Return true if there is a route from "source" to "destination". """ ec2 = self.driver.client("ec2") dest_sg_id, src_sg_id, _, src_ip_permissions = self._extract_service_info( source, destination, port) security_group = ec2.describe_security_groups(GroupIds=[dest_sg_id]) def extract_cidr_port(ip_permissions): cidr_port_list = [] for ip_permission in ip_permissions: if "IpRanges" in ip_permission: for ip_range in ip_permission["IpRanges"]: cidr_port_list.append({ "port": ip_permission["FromPort"], "cidr": ip_range["CidrIp"] }) return cidr_port_list def sg_allowed(ip_permissions, sg_id, port): for ip_permission in ip_permissions: if (ip_permission["UserIdGroupPairs"] and ip_permission["FromPort"] == port): for pair in ip_permission["UserIdGroupPairs"]: if "GroupId" in pair and pair["GroupId"] == sg_id: return True return False ip_permissions = security_group["SecurityGroups"][0]["IpPermissions"] logger.debug("ip_permissions: %s", ip_permissions) if src_sg_id and sg_allowed(ip_permissions, src_sg_id, port): return True src_cidr_port_info = extract_cidr_port(src_ip_permissions) cidr_port_info = extract_cidr_port(ip_permissions) for cidr_port in cidr_port_info: if cidr_port["port"] != port: continue for src_cidr_port in src_cidr_port_info: if (ipaddress.IPv4Network(src_cidr_port["cidr"]).overlaps( ipaddress.IPv4Network(cidr_port["cidr"]))): return True return False
def destroy(self, image): """ Destroy a image given the provided image object. """ ec2 = self.driver.client("ec2") raw_images = ec2.describe_images(Owners=["self"]) logger.debug("Images: %s", raw_images) snapshot_ids = [] for raw_image in raw_images["Images"]: if raw_image["ImageId"] == image.image_id: for bdm in raw_image["BlockDeviceMappings"]: snapshot_id = bdm.get("Ebs", {}).get("SnapshotId") if snapshot_id: snapshot_ids.append(snapshot_id) logger.info("Deregistering image: %s", image.image_id) ec2.deregister_image(ImageId=image.image_id) for snapshot_id in snapshot_ids: logger.info("Deleting snapshot: %s", snapshot_id) ec2.delete_snapshot(SnapshotId=snapshot_id) def retry_if_timeout(exception): """ Checks if this exception is just because we haven't converged yet. """ return isinstance(exception, OperationTimedOut) @retry(wait_fixed=RETRY_DELAY, stop_max_attempt_number=RETRY_COUNT, retry_on_exception=retry_if_timeout) def wait_for_destroyed(image_name): logger.info("Waiting for image: %s to be destroyed", image_name) if self.get(image.name): raise OperationTimedOut( "Timed out waiting for image %s to be gone." % image_name) logger.info("Success, did not find image: %s", image_name) wait_for_destroyed(image.name) return True
def add(self, source, destination, port): """ Adds a route from "source" to "destination". """ logger.debug("Adding path from %s to %s", source, destination) if self.has_access(source, destination, port): logger.info("Service %s already has access to %s on port: %s", source, destination, port) return True # Currently controlling egress in AWS is not supported. All egress is always allowed. if not isinstance(destination, Service): raise DisallowedOperationException( "Destination must be a cloudless.types.networking.Service object" ) dest_sg_id, _, _, src_ip_permissions = self._extract_service_info( source, destination, port) ec2 = self.driver.client("ec2") ec2.authorize_security_group_ingress(GroupId=dest_sg_id, IpPermissions=src_ip_permissions) return Path(destination.network, source, destination, "tcp", port)
def add_instances_to_subnetwork_list(network, service_name, subnetworks): """ Add the instances for this service to the list of subnetworks. Returns the same subnetwork list with the instances added. """ # 1. Get List Of Instances. discovery_retries = 0 discovery_complete = False while discovery_retries < RETRY_COUNT: try: asg = self._discover_asg(network.name, service_name) if not asg: return subnetworks instance_ids = [ instance["InstanceId"] for instance in asg["Instances"] ] instances = discover_instances(instance_ids) logger.debug("Discovered instances: %s", instances) discovery_complete = True except ClientError as client_error: # There is a race between when I discover the autoscaling group # itself and when I try to search for the instances inside it, # so just retry if this happens. logger.debug("Recieved exception discovering instance: %s", client_error) if client_error.response["Error"][ "Code"] == "InvalidInstanceID.NotFound": pass else: raise if discovery_complete: break discovery_retries = discovery_retries + 1 logger.debug("Instance discovery retry number: %s", discovery_retries) if discovery_retries >= RETRY_COUNT: raise OperationTimedOut( "Exceeded retries while discovering %s, in network %s" % (service_name, network)) time.sleep(RETRY_DELAY) # 2. Add instances to subnets. # NOTE: In moto instance objects do not include a "SubnetId" and the IP addresses are # assigned randomly in the VPC, so for now just stripe instances across subnets. if self.mock: for instance, subnetwork, in zip(instances, itertools.cycle(subnetworks)): subnetwork.instances.append( canonicalize_instance_info(instance)) return subnetworks for subnetwork in subnetworks: for instance in instances: if "SubnetId" in instance and subnetwork.subnetwork_id == instance[ "SubnetId"]: subnetwork.instances.append( canonicalize_instance_info(instance)) return subnetworks
def get_blockdev_info(): raw_instance = get_instance(instances[0].instance_id) logger.debug("Getting blockdev info from: %s", raw_instance) if len(raw_instance["BlockDeviceMappings"]) != 1: raise DisallowedOperationException( "Currently only support saving instances with one blockdev, found %s" % (raw_instance)) volume_id = raw_instance["BlockDeviceMappings"][0]["Ebs"][ "VolumeId"] volumes = ec2.describe_volumes(VolumeIds=[volume_id]) if len(volumes["Volumes"]) != 1: raise BadEnvironmentStateException( "Found two volumes with the same id: %s" % volumes) volume = volumes["Volumes"][0] return { "DeviceName": raw_instance["BlockDeviceMappings"][0]["DeviceName"], "Ebs": { "Encrypted": volume["Encrypted"], "DeleteOnTermination": True, "VolumeSize": volume["Size"], "VolumeType": volume["VolumeType"] } }
def wait_for_stopped(instance_id): raw_instance = get_instance(instance_id) logger.debug("Current state: %s", raw_instance) if raw_instance["State"]["Name"] != "stopped": raise OperationTimedOut( "Timed out waiting for instance: %s to stop" % instance_id)
def list(self): """ List all paths and return a dictionary structure representing a graph. """ ec2 = self.driver.client("ec2") sg_to_service = {} for service in self.service.list(): sg_id = self.asg.get_launch_configuration_security_group( service.network.name, service.name) if sg_id in sg_to_service: sg_to_service[sg_id].append(service) else: sg_to_service[sg_id] = [service] security_groups = ec2.describe_security_groups() def make_path(destination, source, rule): return Path(destination.network, source, destination, rule["IpProtocol"], rule.get("FromPort", "N/A")) def get_cidr_paths(destination, ip_permissions): subnets = [] for ip_range in ip_permissions["IpRanges"]: subnets.append( Subnetwork(subnetwork_id=None, name=None, cidr_block=ip_range["CidrIp"], region=None, availability_zone=None, instances=[])) # We treat an explicit CIDR block as a special case of a service with no name. paths = [] if subnets: source = Service(network=None, name=None, subnetworks=subnets) paths.append(make_path(destination, source, ip_permissions)) return paths def get_sg_paths(destination, ip_permissions): paths = [] for group in ip_permissions["UserIdGroupPairs"]: services = sg_to_service[group["GroupId"]] for service in services: paths.append( make_path(destination, service, ip_permissions)) return paths paths = [] for security_group in security_groups["SecurityGroups"]: if security_group["GroupId"] not in sg_to_service: logger.debug( "Security group %s is apparently not attached to a service. Skipping", security_group["GroupId"]) continue services = sg_to_service[security_group["GroupId"]] for service in services: for ip_permissions in security_group["IpPermissions"]: logger.debug("ip_permissions: %s", ip_permissions) paths.extend(get_cidr_paths(service, ip_permissions)) paths.extend(get_sg_paths(service, ip_permissions)) return paths
def destroy(self, service): """ Destroy a group of instances described by "service". """ logger.debug("Attempting to destroy: %s", service) asg_name = AsgName(network=service.network.name, subnetwork=service.name) asg = self._discover_asg(service.network.name, service.name) if asg: self.asg.destroy_auto_scaling_group_instances(asg_name) # Wait for instances to be gone. Need to do this before we can delete # the actual ASG otherwise it will error. def instance_list(service, state): return [ instance for subnetwork in service.subnetworks for instance in subnetwork.instances if instance.state != state ] service = self.get(service.network, service.name) logger.debug("Found service: %s", service) retries = 0 while service and instance_list(service, "terminated"): logger.info( "Waiting for instance termination in service %s. %s still terminating", service.name, len(instance_list(service, "terminated"))) logger.debug("Waiting for instance termination in asg: %s", service) service = self.get(service.network, service.name) retries = retries + 1 if retries > RETRY_COUNT: raise OperationTimedOut("Timed out waiting for ASG scale down") time.sleep(RETRY_DELAY) logger.info("Success! All instances terminated.") asg = self._discover_asg(service.network.name, service.name) if asg: self.asg.destroy_auto_scaling_group(asg_name) # Wait for ASG to be gone. Need to wait for this because it's a dependency of the launch # configuration. asg = self._discover_asg(service.network.name, service.name) retries = 0 while asg: logger.debug("Waiting for asg deletion: %s", asg) asg = self._discover_asg(service.network.name, service.name) retries = retries + 1 if retries > RETRY_COUNT: raise OperationTimedOut("Timed out waiting for ASG deletion") time.sleep(RETRY_DELAY) vpc_id = service.network.network_id lc_security_group = self.asg.get_launch_configuration_security_group( service.network.name, service.name) self.asg.destroy_launch_configuration(asg_name) if lc_security_group: logger.debug("Deleting referencing rules of sg: %s", lc_security_group) self.security_groups.delete_referencing_rules( vpc_id, lc_security_group) logger.debug("Attempting to delete sg: %s", lc_security_group) self.security_groups.delete_with_retries(lc_security_group, RETRY_COUNT, RETRY_DELAY) else: logger.debug("Attempting to delete sg by name: %s", str(asg_name)) self.security_groups.delete_by_name(vpc_id, str(asg_name), RETRY_COUNT, RETRY_DELAY) self.subnetwork.destroy(service.network, service.name)
def get(self, network, service_name): """ Discover a service in "network" named "service_name". """ logger.debug("Discovering autoscaling group named %s in network: %s", service_name, network) def discover_instances(instance_ids): """ Given instance IDs get the actual instance data for them. """ ec2 = self.driver.client("ec2") logger.debug("Discovering instances: %s", instance_ids) instances = {"Reservations": []} # Need this check because if we pass an empty list the API returns all instances if instance_ids: instances = ec2.describe_instances(InstanceIds=instance_ids) return [ instance for reservation in instances["Reservations"] for instance in reservation["Instances"] ] def add_instances_to_subnetwork_list(network, service_name, subnetworks): """ Add the instances for this service to the list of subnetworks. Returns the same subnetwork list with the instances added. """ # 1. Get List Of Instances. discovery_retries = 0 discovery_complete = False while discovery_retries < RETRY_COUNT: try: asg = self._discover_asg(network.name, service_name) if not asg: return subnetworks instance_ids = [ instance["InstanceId"] for instance in asg["Instances"] ] instances = discover_instances(instance_ids) logger.debug("Discovered instances: %s", instances) discovery_complete = True except ClientError as client_error: # There is a race between when I discover the autoscaling group # itself and when I try to search for the instances inside it, # so just retry if this happens. logger.debug("Recieved exception discovering instance: %s", client_error) if client_error.response["Error"][ "Code"] == "InvalidInstanceID.NotFound": pass else: raise if discovery_complete: break discovery_retries = discovery_retries + 1 logger.debug("Instance discovery retry number: %s", discovery_retries) if discovery_retries >= RETRY_COUNT: raise OperationTimedOut( "Exceeded retries while discovering %s, in network %s" % (service_name, network)) time.sleep(RETRY_DELAY) # 2. Add instances to subnets. # NOTE: In moto instance objects do not include a "SubnetId" and the IP addresses are # assigned randomly in the VPC, so for now just stripe instances across subnets. if self.mock: for instance, subnetwork, in zip(instances, itertools.cycle(subnetworks)): subnetwork.instances.append( canonicalize_instance_info(instance)) return subnetworks for subnetwork in subnetworks: for instance in instances: if "SubnetId" in instance and subnetwork.subnetwork_id == instance[ "SubnetId"]: subnetwork.instances.append( canonicalize_instance_info(instance)) return subnetworks # 1. Get List Of subnetworks. The service exists iff this exists. subnetworks = self.subnetwork.get(network, service_name) if not subnetworks: return None # 2. Add instances to subnetworks. subnetworks = add_instances_to_subnetwork_list(network, service_name, subnetworks) # 3. Profit! return Service(network=network, name=service_name, subnetworks=subnetworks)