def client(self, aws_config=None):
"""Represents the VPCConnection Client
"""
aws_config_property = (self._get_aws_config_property(aws_config)
or self._get_aws_config_from_file())
if not aws_config_property:
return VPCConnection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
if 'ec2_region_name' in aws_config:
del (aws_config['ec2_region_name'])
# for backward compatibility,
# delete this key before passing config to Boto
if 'ec2_region_endpoint' in aws_config:
del (aws_config["ec2_region_endpoint"])
return VPCConnection(**aws_config)
def _connect_to_region(self, **kwargs):
if self._isRegionInfo:
return VPCConnection(aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
**kwargs)
for region in self.all_region():
if region.name == self.region:
self.region = region
return VPCConnection(aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region=self.region, **kwargs)
def create_peering_connections(peering_configs):
""" create vpc peering configuration from the peering config dictionary"""
vpc_conn = VPCConnection()
for peering in peering_configs.keys():
vpc_map = peering_configs[peering]['vpc_map']
vpc_metanetwork_map = peering_configs[peering][
'vpc_metanetwork_map']
vpc_ids = [vpc.vpc.id for vpc in vpc_map.values()]
existing_peerings = vpc_conn.get_all_vpc_peering_connections(
filters=[('status-code',
'active'), ('accepter-vpc-info.vpc-id', vpc_ids[0]),
('requester-vpc-info.vpc-id', vpc_ids[1])]
) + vpc_conn.get_all_vpc_peering_connections(
filters=[('status-code',
'active'), ('accepter-vpc-info.vpc-id', vpc_ids[1]),
('requester-vpc-info.vpc-id', vpc_ids[0])])
# create peering when peering doesn't exist
if not existing_peerings:
peering_conn = vpc_conn.create_vpc_peering_connection(*vpc_ids)
vpc_conn.accept_vpc_peering_connection(peering_conn.id)
logging.info("create new peering connection %s for %s",
peering_conn.id, peering)
else:
peering_conn = existing_peerings[0]
logging.info("peering connection %s exists for %s",
existing_peerings[0].id, peering)
DiscoVPC.create_peering_routes(vpc_conn, vpc_map,
vpc_metanetwork_map, peering_conn)
def resolve_security_groups(self):
self.log.info("Resolving security groups")
# If the server is being spun up in a vpc, search only that vpc
exists = lambda s: s in [
group.name for group in self.ec2.get_all_security_groups()
if self.vpc_id == group.vpc_id
]
for index, group in enumerate(self.security_groups):
if not exists(group):
self.log.info('Security Group {group} does not exist'.format(
group=group))
if self.subnet_id is None:
self.ec2.create_security_group(group, group)
else:
vpc_conn = VPCConnection()
vpc_conn.create_security_group(group,
group,
vpc_id=self.vpc_id)
self.log.info(
'Created security group {group}'.format(group=group))
else:
self.log.info('Security Group {group} already exists'.format(
group=group))
def run(self, terms, variables=None, **kwargs):
ret = []
# Below i will implement
valid_lookups = {
'id' : None
'dhcp_options_id' : None
'state' : None
'cidr_block' : None
'is_default' : None
'instance_tenancy' : None
'classic_link_enabled' : None
}
conn = VPCConnection()
for term in terms:
params = term.split(' ')
key_to_lookup = params[0]
try:
assert( key_to_lookup in valid_lookups )
except (AssertionError) as e:
raise AnsibleError(e)
vpc_filter = {}
for param in params[1:]:
tag, value = param.split('=')
vpc_filter.update({'tag:'+ tag : value})
vpcs = conn.get_all_vpcs(None, vpc_filter)
if len(vpcs) > 1:
ret = [ x.get( key_to_lookup ) for x in vpcs ]
return ret
def delete_vpc(region, name):
conn = VPCConnection(region=region)
vpc = get_vpc_by_name(conn, name)
if not vpc:
print "VPC %s does not exist. Nothing to delete" % name
else:
delete_sgs_for_vpc(conn, vpc, name)
delete_route_tables_for_vpc(conn, vpc, name)
delete_igs_for_vpc(conn, vpc, name)
delete_subnets_for_vpc(conn, vpc, name)
print "Deleting VPC %s..." % name
success = False
for i in range(0, 100):
try:
conn.delete_vpc(vpc.id)
success = True
break
except EC2ResponseError as ex:
if ex.error_code == "DependencyViolation":
print "WARN(%s): Sleeping 2 and retrying" % ex.error_code
time.sleep(2)
continue
else:
raise
if not success:
raise Exception("Could not delete vpc")
def aws_connect(creds_fname=None):
credentials_ec2 = creds_fname
if not credentials_ec2:
config = get_cloudsim_config()
# log("config: %s" % config)
credentials_ec2 = config['boto_path']
ec2_region_name, aws_access_key_id, aws_secret_access_key, region_endpoint\
= read_boto_file(credentials_ec2)
if ec2_region_name == 'nova':
# TODO: remove hardcoded OpenStack endpoint
region = RegionInfo(None, 'cloudsim', region_endpoint) # 172.16.0.201
ec2conn = EC2Connection(aws_access_key_id,
aws_secret_access_key,
is_secure=False,
region=region,
port=8773,
path='/services/Cloud')
vpcconn = VPCConnection(aws_access_key_id,
aws_secret_access_key,
is_secure=False,
region=region,
port=8773,
path='/services/Cloud')
else:
region = RegionInfo(None, ec2_region_name, region_endpoint)
ec2conn = boto.connect_ec2(aws_access_key_id,
aws_secret_access_key,
region=region)
vpcconn = boto.connect_vpc(aws_access_key_id,
aws_secret_access_key,
region=region)
return ec2conn, vpcconn
def setUp(self):
"""
Setup method to initialize vpc_connection objectq
"""
super(TestVPCConnection, self).setUp()
self.vpc_connection = VPCConnection(
aws_access_key_id='aws_access_key_id',
aws_secret_access_key='aws_secret_access_key')
def get_subnet2(name, region):
vpc = VPCConnection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
subnet = vpc.get_all_subnets(filters={'tag-value': name})
subnetid = str(subnet)
subnetid = (subnetid.split(":"))[1]
subnetid = subnetid.replace("]", '')
return subnetid
def _configure_environment(self):
"""Create a new disco style environment VPC"""
vpc_cidr = self.get_config("vpc_cidr")
# Create VPC
vpc_conn = VPCConnection()
self.vpc = vpc_conn.create_vpc(self.get_config("vpc_cidr"))
keep_trying(300, self.vpc.add_tag, "Name", self.environment_name)
keep_trying(300, self.vpc.add_tag, "type", self.environment_type)
logging.debug("vpc: %s", self.vpc)
dhcp_options = self._configure_dhcp()
self.vpc.connection.associate_dhcp_options(dhcp_options.id,
self.vpc.id)
# Enable DNS
vpc_conn.modify_vpc_attribute(self.vpc.id, enable_dns_support=True)
vpc_conn.modify_vpc_attribute(self.vpc.id, enable_dns_hostnames=True)
# Create metanetworks (subnets, route_tables and security groups)
for network in self.networks.itervalues():
network.create()
# Configure security group rules
for network in self.networks.values():
self._add_sg_rules(network)
# Set up security group rules
self._open_customer_ports()
# Allow ICMP (ping, traceroute & etc) and DNS traffic for all subnets
for network in self.networks.itervalues():
self.vpc.connection.authorize_security_group(
group_id=network.security_group.id,
ip_protocol="icmp",
from_port=-1,
to_port=-1,
cidr_ip=vpc_cidr)
self.vpc.connection.authorize_security_group(
group_id=network.security_group.id,
ip_protocol="udp",
from_port=53,
to_port=53,
cidr_ip=vpc_cidr)
# Setup internet gateway
internet_gateway = self.vpc.connection.create_internet_gateway()
self.vpc.connection.attach_internet_gateway(internet_gateway.id,
self.vpc.id)
logging.debug("internet_gateway: %s", internet_gateway)
self._add_igw_routes(internet_gateway)
self._attach_vgw()
self.configure_notifications()
DiscoVPC.create_peering_connections(
DiscoVPC.parse_peerings_config(self.vpc.id))
self.rds.update_all_clusters_in_vpc()
def get_vpc(connection, aws_access_key_id, aws_secret_access_key):
global _vpcs
if _vpcs.get(connection.region.name):
return _vpcs[connection.region.name]
_vpcs[connection.region.name] = VPCConnection(
region=connection.region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
return _vpcs[connection.region.name]
def vpc_connect(region_name=config['region']):
region = boto.ec2.get_region(region_name=region_name)
print region
vpcconn = VPCConnection(aws_access_key_id=KEY,
aws_secret_access_key=SECRET,
region=region)
print vpcconn
return vpcconn
def provision_pinger(args):
config = args.config
process_config(config)
aws_config = config["aws_config"]
iam_config = config["iam_config"]
s3_config = config["s3_config"]
vpc_config = config["vpc_config"]
as_config = config["autoscale_config"]
elb_config = config["elb_config"]
misc_config = config["misc"]
load_config_from_s3(s3_config)
print "Provisioning Pinger %s" % vpc_config["name"]
# create connection
conn = VPCConnection(region=aws_config["region"])
# create vpc
try:
create_iam_users_and_policies(aws_config["region_name"],
vpc_config["name"], iam_config)
update_pinger_cfg(config)
upload_pinger_cfg(vpc_config["name"], s3_config)
create_nacho_init_sh(config)
vpc = create_vpc(conn, vpc_config["name"],
vpc_config["vpc_cidr_block"],
vpc_config["instance_tenancy"])
subnet = create_subnet(conn, vpc, vpc_config["name"] + "-SN",
vpc_config["subnet_cidr_block"],
vpc_config["availability_zone"])
ig = create_ig(conn, vpc, vpc_config["name"] + "-IG")
rt = update_route_table(conn, vpc, ig, vpc_config["name"] + "-RT")
elb_sg_config = config["elb_config"]["sg_config"]
elb_sg = create_sg(
conn, vpc, vpc_config["name"] + elb_sg_config["name"] + "-SG",
elb_sg_config["description"] + " for " + vpc_config["name"])
add_rules_to_sg(conn, elb_sg, elb_sg_config["ingress-rules"], True,
misc_config["jumphost"])
add_rules_to_sg(conn, elb_sg, elb_sg_config["egress-rules"], False,
misc_config["jumphost"])
elb = create_elb(aws_config["region_name"], vpc, subnet, elb_sg,
vpc_config["name"] + "-ELB", elb_config,
s3_config["first_cert_pem"])
ins_sg_config = config["autoscale_config"]["sg_config"]
ins_sg = create_sg(
conn, vpc, vpc_config["name"] + ins_sg_config["name"] + "-SG",
ins_sg_config["description"] + " for " + vpc_config["name"])
add_rules_to_sg(conn, ins_sg, ins_sg_config["ingress-rules"], True,
misc_config["jumphost"])
add_rules_to_sg(conn, ins_sg, ins_sg_config["egress-rules"], False,
misc_config["jumphost"])
ascaler = create_autoscaler(aws_config["region_name"], vpc, elb,
subnet, ins_sg, vpc_config["name"] + "-AS",
aws_config, as_config)
except (BotoServerError, S3ResponseError, EC2ResponseError) as e:
print "Error :%s(%s):%s" % (e.error_code, e.status, e.message)
print traceback.format_exc()
cleanup(config)
def init_connection(conf):
# Load general configuration file
cnf = ConfigParser()
try:
cnf.read(conf)
region_os = EC2RegionInfo(endpoint=cnf.get('outscale', 'endpoint'))
ows = VPCConnection(cnf.get('outscale', 'access_key'), cnf.get('outscale', 'secret_key'), region=region_os)
return ows
except Exception as e:
print 'error in the configuration file: ', e
return None
def delete_peerings(vpc_id=None):
"""Delete peerings. If vpc_id is specified, delete all peerings of the VPCs only"""
vpc_conn = VPCConnection()
for peering in DiscoVPC.list_peerings(vpc_id):
try:
logging.info('deleting peering connection %s', peering.id)
vpc_conn.delete_vpc_peering_connection(peering.id)
except EC2ResponseError:
raise RuntimeError(
'Failed to delete VPC Peering connection {}'.format(
peering.id))
def setUpClass(self):
time_str = str(int(time.time()))
self.route53 = Route53Connection()
self.base_domain = 'boto-private-zone-test-%s.com' % time_str
self.vpc = VPCConnection()
self.test_vpc = self.vpc.create_vpc(cidr_block='10.11.0.0/16')
# tag the vpc to make it easily identifiable if things go spang
self.test_vpc.add_tag("Name", self.base_domain)
self.zone = self.route53.get_zone(self.base_domain)
if self.zone is not None:
self.zone.delete()
def get_subnet_vpc_id(self, subnet_id):
vpc_conn = VPCConnection()
subnets = vpc_conn.get_all_subnets(filters={'subnet_id': subnet_id})
if len(subnets) == 1:
vpc_id = subnets[0].vpc_id
return vpc_id
elif len(subnets) == 0:
raise NoSubnetReturned(
"No subnets returned for: {}".format(subnet_id))
else:
raise Exception("More than 1 subnet returned")
def find_vpc_id_by_name(vpc_name):
"""Find VPC by name"""
vpc_conn = VPCConnection()
vpc_ids = vpc_conn.get_all_vpcs(filters={'tag:Name': vpc_name})
if len(vpc_ids) == 1:
return vpc_ids[0].id
elif len(vpc_ids) == 0:
raise VPCNameNotFound("No VPC is named as {}".format(vpc_name))
else:
raise MultipleVPCsForVPCNameError(
"More than 1 VPC is named as {}".format(vpc_name))
def test_db_subnet_group(self):
vpc_api = VPCConnection()
rds_api = RDSConnection()
vpc = vpc_api.create_vpc('10.0.0.0/16')
az_list = vpc_api.get_all_zones(filters={'state': 'available'})
subnet = list()
n = 0
for az in az_list:
try:
subnet.append(
vpc_api.create_subnet(vpc.id,
'10.0.' + str(n) + '.0/24',
availability_zone=az.name))
n = n + 1
except:
pass
grp_name = 'db_subnet_group' + str(int(time.time()))
subnet_group = rds_api.create_db_subnet_group(
grp_name, grp_name, [subnet[0].id, subnet[1].id])
if not _is_ok(subnet_group, vpc.id, grp_name,
[subnet[0].id, subnet[1].id]):
raise Exception("create_db_subnet_group returned bad values")
rds_api.modify_db_subnet_group(grp_name, description='new description')
subnet_grps = rds_api.get_all_db_subnet_groups(name=grp_name)
if not _is_ok(subnet_grps[0], vpc.id, 'new description',
[subnet[0].id, subnet[1].id]):
raise Exception(
"modifying the subnet group desciption returned bad values")
rds_api.modify_db_subnet_group(grp_name,
subnet_ids=[subnet[1].id, subnet[2].id])
subnet_grps = rds_api.get_all_db_subnet_groups(name=grp_name)
if not _is_ok(subnet_grps[0], vpc.id, 'new description',
[subnet[1].id, subnet[2].id]):
raise Exception(
"modifying the subnet group subnets returned bad values")
rds_api.delete_db_subnet_group(subnet_group.name)
try:
rds_api.get_all_db_subnet_groups(name=grp_name)
raise Exception(subnet_group.name +
" still accessible after delete_db_subnet_group")
except:
pass
while n > 0:
n = n - 1
vpc_api.delete_subnet(subnet[n].id)
vpc_api.delete_vpc(vpc.id)
def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs):
"""
:type aws_access_key_id: string
:param aws_access_key_id: Your AWS Access Key ID
:type aws_secret_access_key: string
:param aws_secret_access_key: Your AWS Secret Access Key
:rtype: :class:`boto.vpc.VPCConnection`
:return: A connection to VPC
"""
from boto.vpc import VPCConnection
return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs)
def __init__(self, name, vpc, network_cidr=None, boto3_connection=None):
self.vpc = vpc
self.name = name
if network_cidr:
self._network_cidr = IPNetwork(network_cidr)
else:
self._network_cidr = None
self._centralized_route_table_loaded = False
self._centralized_route_table = None # lazily initialized
self._security_group = None # lazily initialized
self._connection = VPCConnection()
self._disco_subnets = None # lazily initialized
self._boto3_connection = boto3_connection # Lazily initialized if parameter is None
def remove_vpc(self):
c = VPCConnection()
for subnet in c.get_all_subnets():
if subnet.tags.get('organization') == self.organization:
c.delete_subnet(subnet.id)
for vpc in c.get_all_vpcs():
if vpc.tags.get('organization') == self.organization:
vpc.delete()
db_vpc = organization_vpc.OrganizationVPC()
org = organizations.Organizations()
org.load(name=self.organization_name)
db_vpc.load(organization_id=org.get('id'))
db_vpc.delete()
def create_nacl(vpc_id, region, network_aclname):
vpc = VPCConnection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
network = vpc.create_network_acl(vpc_id)
time.sleep(1)
ec2_conn = boto.ec2.connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_conn.create_tags(network.id, {
"Name": network_aclname,
"Project": 'LiveLiveProd'
})
return network.id
def launch_tag(tag_name,
version=keys.v2,
reservation_delay=5,
pending_delay=5,
completion_delay=180):
print 'launch_tag:', tag_name
conn = boto.ec2.connect_to_region("us-east-1")
worker_id = [
sg.id for sg in conn.get_all_security_groups() if sg.name == 'worker'
][0]
print 'security group:', worker_id
image_id = conn.get_all_images(owners=['self'], filters={'name':
version})[0].id
print 'ami:', image_id
subnet_id = VPCConnection().get_all_subnets()[0].id
print 'subnet:', subnet_id
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
subnet_id=subnet_id,
groups=[worker_id],
associate_public_ip_address=True)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(
interface)
_it = get_size(tag_name)
print 'instance type:', _it
reservation = conn.run_instances(image_id=image_id,
instance_type=_it,
network_interfaces=interfaces,
key_name='lucid')
print 'reservation:', reservation
time.sleep(reservation_delay)
instance = reservation.instances[0]
print 'instance:', instance
instance.add_tag("Name", tag_name)
instance.update()
print 'update tag Name:', tag_name
while instance.state == "pending":
print 'instance state:', instance, instance.state
time.sleep(pending_delay)
instance.update()
print 'launch done', instance
time.sleep(completion_delay)
def get_subnet_availability_zone(self, subnet_id):
self.log.info(
"getting zone for subnet {subnet_id}".format(subnet_id=subnet_id))
vpc_conn = VPCConnection()
filters = {'subnet-id': subnet_id}
subnets = vpc_conn.get_all_subnets(filters=filters)
if len(subnets) == 1:
availability_zone = subnets[0].availability_zone
log_message = 'Subnet {subnet_id} is in ' \
'availability zone {availability_zone}'
self.log.info(log_message.format(
subnet_id=subnet_id,
availability_zone=availability_zone))
return availability_zone
def create_subnet(vpc_id, cidr_block, availability_zone, subnet_name, region):
vpc = VPCConnection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
datacenters = vpc.create_subnet(vpc_id=vpc_id,
cidr_block=cidr_block,
availability_zone=availability_zone)
ec2_conn = boto.ec2.connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
time.sleep(1)
ec2_conn.create_tags(datacenters.id, {
"Name": subnet_name,
"Project": 'LiveLiveProd'
})
return datacenters.id
def fetch_environment(cls, vpc_id=None, environment_name=None):
"""
Returns an instance of this class for the specified VPC, or None if it does not exist
"""
vpc_conn = VPCConnection()
if vpc_id:
vpc = vpc_conn.get_all_vpcs(vpc_ids=[vpc_id])
elif environment_name:
vpc = vpc_conn.get_all_vpcs(filters={"tag:Name": environment_name})
else:
raise VPCEnvironmentError("Expect vpc_id or environment_name")
if vpc:
vpc = vpc[0]
return cls(vpc.tags["Name"], vpc.tags["type"], vpc)
else:
return None
def parse_peerings_config(vpc_id=None):
"""
Parses configuration from disco_vpc.ini's peerings sections.
If vpc_id is specified, only configuration relevant to vpc_id is included.
"""
logging.debug("Parsing peerings configuration specified in %s",
CONFIG_FILE)
vpc_conn = VPCConnection()
config = read_config(CONFIG_FILE)
if 'peerings' not in config.sections():
logging.info("No VPC peering configuration defined.")
return {}
peerings = [
peering[1] for peering in config.items('peerings')
if peering[0].startswith('connection_')
]
for peering in peerings:
endpoints = [_.strip() for _ in peering.split(' ')]
if len(endpoints) != 2:
raise VPCPeeringSyntaxError(
"Syntax error in vpc peering connection. "
"Expected 2 space-delimited endpoints but found: '{}'".
format(peering))
peering_configs = {}
for peering in peerings:
peering_config = DiscoVPC.parse_peering_connection_line(
peering, vpc_conn)
vpc_ids_in_peering = [
vpc.vpc.id
for vpc in peering_config.get("vpc_map", {}).values()
]
if len(vpc_ids_in_peering) < 2:
pass # not all vpcs were up, nothing to do
elif vpc_id and vpc_id not in vpc_ids_in_peering:
logging.debug(
"Skipping peering %s because it doesn't include %s",
peering, vpc_id)
else:
peering_configs[peering] = peering_config
return peering_configs
def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs):
"""
:type aws_access_key_id: string
:param aws_access_key_id: Your AWS Access Key ID
:type aws_secret_access_key: string
:param aws_secret_access_key: Your AWS Secret Access Key
:rtype: :class:`boto.vpc.VPCConnection`
:return: A connection to VPC
"""
access_key, secret_key = get_govcloud_creds(aws_access_key_id,
aws_secret_access_key)
region = RegionInfo(name='govcloud',
endpoint='ec2.us-gov-west-1.amazonaws.com')
from boto.vpc import VPCConnection
return VPCConnection(access_key, secret_key,
region=region, **kwargs)
def _get_vpc_details(self):
"""
Create dictonary with VPC's info and tags
"""
# Connect to AWS for VPC
vpc_conn = VPCConnection()
# Grab VPCs
vpcs_data = vpc_conn.get_all_vpcs()
for vpc in vpcs_data:
tags = {}
tags['state'] = vpc.state
tags['cidr'] = vpc.cidr_block
for key, value in vpc.tags.items():
tags[key.lower()] = value.lower()
# assing vpcs to class vpcs
self.vpcs = {vpc.id: tags}
