def get_or_create(self, config):
"""Get or create Aws Ec2 vpcs
Args:
config (dict): Vpcs config
Returns:
dict: vpcs configs
"""
created_vpcs = {}
for index, vpc_config in enumerate(config):
vpcs = filter_resources(self.ec2.vpcs, "cidrBlock",
vpc_config["CidrBlock"])
if not vpcs:
vpc = self.ec2.create_vpc(CidrBlock=vpc_config["CidrBlock"])
else:
vpc = self.ec2.Vpc(vpcs[0].id)
self.logger.info(
"A vpc with ID '%s' and cidr block '%s' has been created or already exists",
vpc.vpc_id, vpc_config["CidrBlock"])
tag_with_name_with_suffix(vpc, "vpc", index, self.tag_base_name)
vpc_config["VpcId"] = vpc.vpc_id
created_vpcs[vpc.vpc_id] = vpc_config
return created_vpcs
def create(self, config):
"""Create network interfaces
Args:
config (dict): Vpcs config
"""
for vpc_id, vpc_config in config.iteritems():
index = 0
for subnet in vpc_config["Subnets"]:
aws_subnet = self.ec2.Subnet(subnet["SubnetId"])
for eni in subnet["NetworkInterfaces"]:
found_eni = filter_resources(
aws_subnet.network_interfaces, "tag-value", eni["uid"])
if not found_eni:
created_eni = aws_subnet.create_network_interface(
SecondaryPrivateIpAddressCount=eni["Ips"][
"SecondaryPrivateIpAddressCount"],
)
tag_with_name_with_suffix(
created_eni, "eni", index, self.tag_base_name)
created_eni.create_tags(
Tags=[{
"Key": "uid",
"Value": eni["uid"]
}]
)
index = index + 1
self.logger.info("A network interface '%s' for subnet '%s' has been created or already exists",
eni["uid"],
subnet["SubnetId"]
)
def get_or_create(self, config):
"""Get or create route tables
Args:
config (dict): Vpcs config
Returns:
dict: Security groups configs
"""
created_route_tables = []
index = 0
for vpc_id, vpc_config in config.iteritems():
route_tables = filter_resources(self.ec2.route_tables, "vpc-id",
vpc_id)
if not route_tables:
route_table = self.ec2.create_route_table(VpcId=vpc_id)
else:
route_table = self.ec2.RouteTable(route_tables[0].id)
self.logger.info(
"A route table " +
"with ID '%s' and attached to vpc '%s' has been created or already exists",
route_table.id, vpc_id)
tag_with_name_with_suffix(route_table, "rt", index,
self.tag_base_name)
created_route_tables.append({"RouteTableId": route_table.id})
index = index + 1
return {vpc_config["VpcId"]: {"RouteTables": created_route_tables}}
def get_or_create(self, config):
"""Get or create Aws Ec2 vpcs
Args:
config (dict): Vpcs config
Returns:
dict: vpcs configs
"""
created_vpcs = {}
for index, vpc_config in enumerate(config):
vpcs = filter_resources(
self.ec2.vpcs, "cidrBlock", vpc_config["CidrBlock"])
if not vpcs:
vpc = self.ec2.create_vpc(CidrBlock=vpc_config["CidrBlock"])
else:
vpc = self.ec2.Vpc(vpcs[0].id)
self.logger.info("A vpc with ID '%s' and cidr block '%s' has been created or already exists",
vpc.vpc_id,
vpc_config["CidrBlock"]
)
tag_with_name_with_suffix(
vpc, "vpc", index, self.tag_base_name)
vpc_config["VpcId"] = vpc.vpc_id
created_vpcs[vpc.vpc_id] = vpc_config
return created_vpcs
def release_public_ips(self):
"""Dissociate public ips to elastic network interfaces and release ips
"""
aws_enis = filter_resources(
self.ec2.network_interfaces, "tag:Name", self.tag_base_name + '-*')
eni_ids = []
for aws_eni in aws_enis:
for aws_private_ip_address in aws_eni.private_ip_addresses:
eni_ids.append(aws_eni.id)
aws_addresses = self.ec2_client.describe_addresses(
Filters=[
{
'Name': 'network-interface-id',
'Values': eni_ids
},
],
)
for aws_public_ip in aws_addresses['Addresses']:
self.ec2_client.disassociate_address(
AssociationId=aws_public_ip['AssociationId']
)
self.ec2_client.release_address(
AllocationId=aws_public_ip['AllocationId'],
)
self.logger.info(
"The public IP '%s' has been dissociated from network interface '%s' and released",
aws_public_ip['PublicIp'],
aws_public_ip['NetworkInterfaceId']
)
def get_or_create(self, config):
"""Get or create Aws Ec2 security groups
Args:
config (dict): Vpcs config
Returns:
dict: Security groups configs
"""
created_security_groups = []
for vpc_id, vpc_config in config.iteritems():
if "SecurityGroups" in vpc_config:
for index, sg in enumerate(vpc_config["SecurityGroups"]):
security_groups = filter_resources(
self.ec2.security_groups, "vpc-id", vpc_config["VpcId"])
if not security_groups:
resource = self.ec2.create_security_group(
VpcId=vpc_config["VpcId"],
GroupName=sg["GroupName"],
Description=sg["Description"])
else:
resource = self.ec2.SecurityGroup(
security_groups[0].id)
if "IngressRules" in sg:
self.authorize_sg_ingress_rules(resource, sg)
if "EgressRules" in sg:
self.authorize_sg_egress_rules(resource, sg)
self.logger.info(
"A security group with group name '%s', " +
"with ID '%s' and attached to vpc '%s' has been created or already exists",
sg["GroupName"],
resource.id,
vpc_config["VpcId"]
)
tag_with_name_with_suffix(
resource, "sg", index, self.tag_base_name)
created_security_groups.append(
{
"SecurityGroupId": resource.id,
"GroupName": sg["GroupName"],
"Description": sg["Description"],
}
)
return {
vpc_config["VpcId"]: {
"SecurityGroups": created_security_groups
}
}
return created_security_groups
def get_or_create(self, config):
"""Get or create internet gateways
Args:
config (dict): config config
Returns:
dict: Internet gateways config
"""
created_resources = []
index = 0
for vpc_id, vpc_config in config.iteritems():
if "CreateInternetGateway" in vpc_config:
internet_gateways = filter_resources(
self.ec2.internet_gateways, "attachment.vpc-id", vpc_config["VpcId"])
if not internet_gateways:
internet_gateway = self.ec2.create_internet_gateway()
self.ec2.Vpc(vpc_config["VpcId"]).attach_internet_gateway(
InternetGatewayId=internet_gateway.id,
)
else:
internet_gateway = self.ec2.InternetGateway(internet_gateways[0].id)
for attachment in internet_gateway.attachments:
vpc_already_attached = False
if attachment["VpcId"] == vpc_config["VpcId"]:
vpc_already_attached = True
if not vpc_already_attached:
self.ec2.Vpc(vpc_config["VpcId"]).attach_internet_gateway(
InternetGatewayId=internet_gateway.id,
)
tag_with_name_with_suffix(
internet_gateway, "ig", index, self.tag_base_name)
self.logger.info(
"An internet gateway with ID '%s' attached to vpc '%s' has been created or already exists",
internet_gateway.id,
vpc_config["VpcId"]
)
created_resources.append(
{
"InternetGatewayId": internet_gateway.id,
}
)
index = index + 1
return {
vpc_config["VpcId"]: {
"InternetGateways": created_resources
}
}
def delete(self):
"""Delete Vpcs
"""
vpcs = filter_resources(self.ec2.vpcs, "tag:Name",
self.tag_base_name + '-*')
for vpc in vpcs:
vpc.delete()
self.logger.info(
"The vpc with ID '%s' has been deleted ",
vpc.id,
)
def delete(self):
subnets = filter_resources(
self.ec2.subnets,
"tag:Name",
self.tag_base_name + '-*'
)
for subnet in subnets:
subnet.delete()
self.logger.info(
"The subnet with ID '%s' has been deleted ",
subnet.id,
)
def associate_subnets_to_routes(self, config):
"""Associate subnets to routes
Args:
config (dict): Vpcs config
"""
for vpc_id, vpc_config in config.iteritems():
for route in vpc_config["RouteTables"]:
route_resource = self.ec2.RouteTable(route["RouteTableId"])
for subnet in vpc_config["Subnets"]:
found_associations = filter_resources(
self.ec2.route_tables, "association.subnet-id", subnet["SubnetId"])
if not found_associations:
route_resource.associate_with_subnet(
SubnetId=subnet["SubnetId"])
def delete(self):
"""Delete Vpcs
"""
vpcs = filter_resources(
self.ec2.vpcs,
"tag:Name",
self.tag_base_name + '-*'
)
for vpc in vpcs:
vpc.delete()
self.logger.info(
"The vpc with ID '%s' has been deleted ",
vpc.id,
)
def delete(self):
"""Delete security groups
"""
security_groups = filter_resources(
self.ec2.security_groups,
"tag:Name",
self.tag_base_name + '-*'
)
for security_group in security_groups:
if "default" != security_group.group_name:
security_group.delete()
self.logger.info(
"The security group with ID '%s' has been deleted ",
security_group.id,
)
def delete(self):
"""Delete network acls
"""
network_acls = filter_resources(
self.ec2.network_acls,
"tag:Name",
self.tag_base_name + '-*'
)
for network_acl in network_acls:
if not network_acl.is_default:
network_acl.delete()
self.logger.info(
"The network acl with ID '%s' has been deleted ",
network_acl.id,
)
def get_or_create(self, config):
"""Get or create network acls
Args:
config (object): Vpcs config
Returns:
object: Network acl config
"""
created_network_acls = []
index = 0
for vpc_id, vpc_config in config.iteritems():
network_acls = filter_resources(
self.ec2.network_acls, "vpc-id", vpc_id)
if not network_acls:
network_acl = self.ec2.create_network_acl(
VpcId=vpc_id
)
network_acl.id
else:
network_acl = self.ec2.NetworkAcl(network_acls[0].id)
self.logger.info(
"A network acl " +
"with ID '%s' and attached to pvc '%s' has been created or already exists",
network_acl.id,
vpc_id
)
tag_with_name_with_suffix(
network_acl, "netacl", index, self.tag_base_name)
created_network_acls.append(
{
"NetworkAclId": network_acl.id,
}
)
index = index + 1
return {
vpc_config["VpcId"]: {
"NetworkAcls": created_network_acls
}
}
def associate_subnets_to_routes(self, config):
"""Associate subnets to routes
Args:
config (dict): Vpcs config
"""
for vpc_id, vpc_config in config.iteritems():
for route in vpc_config["RouteTables"]:
route_resource = self.ec2.RouteTable(route["RouteTableId"])
for subnet in vpc_config["Subnets"]:
found_associations = filter_resources(
self.ec2.route_tables, "association.subnet-id",
subnet["SubnetId"])
if not found_associations:
route_resource.associate_with_subnet(
SubnetId=subnet["SubnetId"])
def get_or_create(self, config):
"""Get or create Aws Ec2 subnets
Args:
config (dict): Vpcs config
Returns:
dict: Subnets configs
"""
created_subnets = []
for vpc_id, vpc_config in config.iteritems():
if "Subnets" in vpc_config:
for index, subnet_config in enumerate(vpc_config["Subnets"]):
subnets = filter_resources(
self.ec2.subnets, "cidrBlock", subnet_config["CidrBlock"])
if not subnets:
subnet = self.ec2.create_subnet(
VpcId=vpc_config["VpcId"], CidrBlock=subnet_config["CidrBlock"])
else:
subnet = self.ec2.Subnet(subnets[0].id)
self.logger.info(
"A subnet with ID '%s', " +
"cidr block '%s' and attached to vpc '%s' has been created or already exists",
subnet.id,
subnet_config["CidrBlock"],
vpc_config["VpcId"]
)
tag_with_name_with_suffix(
subnet, "subnet", index, self.tag_base_name)
created_subnets.append(
{
"SubnetId": subnet.id,
"CidrBlock": subnet_config["CidrBlock"],
"NetworkInterfaces": subnet_config["NetworkInterfaces"]
}
)
return {
vpc_config["VpcId"]: {
"Subnets": created_subnets
}
}
def delete(self):
"""Delete route tables
"""
route_tables = filter_resources(
self.ec2.route_tables,
"tag:Name",
self.tag_base_name + '-*'
)
for route_table in route_tables:
is_main_route_table = True
if hasattr(route_table, 'associations'):
for association in route_table.associations.all():
if not association.main:
is_main_route_table = False
self.ec2_client.disassociate_route_table(
AssociationId=association.id
)
self.logger.info(
"The route table association with ID '%s' has been deleted",
association.id,
)
for route in route_table.routes:
if 'local' != route['GatewayId']:
self.ec2_client.delete_route(
RouteTableId=route_table.id,
DestinationCidrBlock=route['DestinationCidrBlock']
)
self.logger.info(
"The route for gateway ID '%s' with cird block '%s' and " +
"associated to route table '%s' has been deleted",
route['GatewayId'],
route['DestinationCidrBlock'],
route_table.id
)
if not is_main_route_table:
route_table.delete()
self.logger.info(
"The route table with ID '%s' has been deleted",
route_table.id
)
def delete(self):
"""Delete elastic network interfaces
"""
aws_enis = filter_resources(
self.ec2.network_interfaces, "tag:Name", self.tag_base_name + '-*')
for aws_eni in aws_enis:
if hasattr(aws_eni, 'attachment') and aws_eni.attachment is not None:
print aws_eni.attachment
self.ec2_client.detach_network_interface(
AttachmentId=aws_eni.attachment['AttachmentId'],
)
aws_eni.delete()
self.logger.info(
"The network interface '%s' has been detached from instance and deleted",
aws_eni.id
)
def get_or_create(self, config):
"""Get or create route tables
Args:
config (dict): Vpcs config
Returns:
dict: Security groups configs
"""
created_route_tables = []
index = 0
for vpc_id, vpc_config in config.iteritems():
route_tables = filter_resources(
self.ec2.route_tables, "vpc-id", vpc_id)
if not route_tables:
route_table = self.ec2.create_route_table(VpcId=vpc_id)
else:
route_table = self.ec2.RouteTable(route_tables[0].id)
self.logger.info(
"A route table " +
"with ID '%s' and attached to vpc '%s' has been created or already exists",
route_table.id,
vpc_id
)
tag_with_name_with_suffix(
route_table, "rt", index, self.tag_base_name)
created_route_tables.append(
{
"RouteTableId": route_table.id
}
)
index = index + 1
return {
vpc_config["VpcId"]: {
"RouteTables": created_route_tables
}
}
def delete(self):
"""Delete internet gateways
"""
internet_gateways = filter_resources(
self.ec2.internet_gateways,
"tag:Name",
self.tag_base_name + '-*'
)
for internet_gateway in internet_gateways:
if hasattr(internet_gateway, 'attachments'):
for attachment in internet_gateway.attachments:
internet_gateway.detach_from_vpc(VpcId=attachment['VpcId'])
internet_gateway.delete()
self.logger.info(
"The internet_gateway with ID '%s' has been deleted ",
internet_gateway.id,
)
def delete(self):
"""Delete route tables
"""
route_tables = filter_resources(self.ec2.route_tables, "tag:Name",
self.tag_base_name + '-*')
for route_table in route_tables:
is_main_route_table = True
if hasattr(route_table, 'associations'):
for association in route_table.associations.all():
if not association.main:
is_main_route_table = False
self.ec2_client.disassociate_route_table(
AssociationId=association.id)
self.logger.info(
"The route table association with ID '%s' has been deleted",
association.id,
)
for route in route_table.routes:
if 'local' != route['GatewayId']:
self.ec2_client.delete_route(
RouteTableId=route_table.id,
DestinationCidrBlock=route['DestinationCidrBlock'])
self.logger.info(
"The route for gateway ID '%s' with cird block '%s' and "
+ "associated to route table '%s' has been deleted",
route['GatewayId'], route['DestinationCidrBlock'],
route_table.id)
if not is_main_route_table:
route_table.delete()
self.logger.info(
"The route table with ID '%s' has been deleted",
route_table.id)
def create(self, instances_groups_config, vpcs_config):
"""Create instances
Args:
instances_groups_config (dict): Instances groups config
vpcs_config (dict): Vpcs config
"""
instances_config = []
user_data = "#!/bin/bash\n"
for instance_group in instances_groups_config:
for instance_index, instance in enumerate(instance_group['Instances']):
instance_config = {
'ImageId': instance_group['ImageId'],
'MinCount': 1,
'MaxCount': 1,
'InstanceType': instance_group['InstanceType'],
'DisableApiTermination': False,
'InstanceInitiatedShutdownBehavior': 'terminate',
'NetworkInterfaces': [],
}
for index, eni in enumerate(instance['NetworkInterfaces']):
found_eni = filter_resources(self.ec2.network_interfaces, "tag-value", eni["uid"])
if found_eni:
instance_config['NetworkInterfaces'].append({
'NetworkInterfaceId': found_eni[0].id,
'DeviceIndex': index
})
if index > 0:
user_data += "\n\nsudo bash -c \"echo 'auto eth{0}' >> /etc/network/interfaces\"\n" \
"sudo bash -c \"echo 'iface eth{0} inet dhcp' >> /etc/network/interfaces\"\n" \
"sudo ifup eth{0}\n" \
"sudo bash -c \"echo '40{0} eth{0}_rt' >> /etc/iproute2/rt_tables\"\n".format(index)
for private_ip_address in found_eni[0].private_ip_addresses:
if not private_ip_address['Primary']:
user_data += "\n# Add the primary ip address to the network interface\n"
user_data += "sudo ip addr add {0}{1} dev eth{2}\n".format(
private_ip_address['PrivateIpAddress'], instance['SubnetCidrSuffix'], index
)
if index > 0:
user_data += "\n# Add an ip rule to a routing table\n"
user_data += "sudo ip rule add from {0} lookup eth{1}_rt\n".format(
private_ip_address['PrivateIpAddress'],
index
)
if index > 0:
user_data += "\n# Add a route\n"
user_data += "sudo ip route add default via {0} dev " \
"eth{1} table eth{1}_rt\n".format(instance['GatewayIP'], index)
instance_config['UserData'] = user_data
aws_reservation = self.ec2_client.run_instances(**instance_config)
aws_instance_config = aws_reservation['Instances'][0]
aws_instance = self.ec2.Instance(aws_instance_config['InstanceId'])
tag_with_name_with_suffix(aws_instance, "i", instance_index, self.tag_base_name)
instance_config['InstanceId'] = aws_instance_config['InstanceId']
instances_config.append(instance_config)
return instances_config
def create(self, instances_groups_config, vpcs_config):
"""Create instances
Args:
instances_groups_config (dict): Instances groups config
vpcs_config (dict): Vpcs config
"""
instances_config = []
user_data = "#!/bin/bash\n"
for instance_group in instances_groups_config:
for instance_index, instance in enumerate(
instance_group['Instances']):
instance_config = {
'ImageId': instance_group['ImageId'],
'MinCount': 1,
'MaxCount': 1,
'InstanceType': instance_group['InstanceType'],
'DisableApiTermination': False,
'InstanceInitiatedShutdownBehavior': 'terminate',
'NetworkInterfaces': [],
}
for index, eni in enumerate(instance['NetworkInterfaces']):
found_eni = filter_resources(self.ec2.network_interfaces,
"tag-value", eni["uid"])
if found_eni:
instance_config['NetworkInterfaces'].append({
'NetworkInterfaceId':
found_eni[0].id,
'DeviceIndex':
index
})
if index > 0:
user_data += "\n\nsudo bash -c \"echo 'auto eth{0}' >> /etc/network/interfaces\"\n" \
"sudo bash -c \"echo 'iface eth{0} inet dhcp' >> /etc/network/interfaces\"\n" \
"sudo ifup eth{0}\n" \
"sudo bash -c \"echo '40{0} eth{0}_rt' >> /etc/iproute2/rt_tables\"\n".format(index)
for private_ip_address in found_eni[
0].private_ip_addresses:
if not private_ip_address['Primary']:
user_data += "\n# Add the primary ip address to the network interface\n"
user_data += "sudo ip addr add {0}{1} dev eth{2}\n".format(
private_ip_address['PrivateIpAddress'],
instance['SubnetCidrSuffix'], index)
if index > 0:
user_data += "\n# Add an ip rule to a routing table\n"
user_data += "sudo ip rule add from {0} lookup eth{1}_rt\n".format(
private_ip_address['PrivateIpAddress'], index)
if index > 0:
user_data += "\n# Add a route\n"
user_data += "sudo ip route add default via {0} dev " \
"eth{1} table eth{1}_rt\n".format(instance['GatewayIP'], index)
instance_config['UserData'] = user_data
aws_reservation = self.ec2_client.run_instances(
**instance_config)
aws_instance_config = aws_reservation['Instances'][0]
aws_instance = self.ec2.Instance(
aws_instance_config['InstanceId'])
tag_with_name_with_suffix(aws_instance, "i", instance_index,
self.tag_base_name)
instance_config['InstanceId'] = aws_instance_config[
'InstanceId']
instances_config.append(instance_config)
return instances_config