def create_vpc(self, cidr_block):
self.logger.debug("start to create VPC, CIDR = %s" % cidr_block)
for i in range(self.RETRY):
try:
vpc = self.conn.create_vpc(cidr_block)
vpc.add_tag('Name', self.TAG_NAME)
self.logger.info("successfully to create VPC id = %s" % vpc.id)
self.vpc = vpc
return self.vpc.id
except Exception, e:
self.logger.error("system error = %s, retry..." % e)
time.sleep(self.RETRY_WAIT)
def create_vpc(conn, name, cidr_block, instance_tenancy):
print "Creating vpc %s" % name
vpc = get_vpc_by_name(conn, name)
if not vpc:
vpc = conn.create_vpc(cidr_block, instance_tenancy=instance_tenancy)
print "Created VPC %s (%s) for cidr_block: %s" % (name, vpc.id,
vpc.cidr_block)
wait_for_vpc(conn, vpc.id)
vpc.add_tag("Name", name)
else:
print "VPC %s (%s) already exists at cidr_block: %s!" % (
name, vpc.id, vpc.cidr_block)
return vpc
def setup_module():
mock.start()
l = logging.getLogger('mock_network')
# create the network
for region, vpcs in NETWORK:
conn = boto.vpc.connect_to_region(region)
l.info("Setting up region {}".format(region))
sgids_by_name = {}
for name, kwargses in SECURITY_GROUPS:
sg = conn.create_security_group(name, 'd:{}'.format(name))
sgids_by_name[name] = sg.id
l.info("Security group {} has id {}".format(name, sg.id))
for kwargs in kwargses:
conn.authorize_security_group(name, **kwargs)
for name, vpc_cidr, subnets in vpcs:
vpc_cidr_ip = IP(vpc_cidr)
vpc = conn.create_vpc(vpc_cidr)
vpc.add_tag('Name', name)
l.info("VPC {} has id {}".format(name, vpc.id))
for name, subnet_cidr, instances in subnets:
subnet_cidr_ip = IP(subnet_cidr)
assert subnet_cidr_ip in vpc_cidr_ip
subnet = conn.create_subnet(vpc.id, subnet_cidr)
subnet.add_tag('Name', name)
l.info("Subnet {} has id {}".format(name, subnet.id))
for name, address, sgs in instances:
address_ip = IP(address)
assert address_ip in subnet_cidr_ip
sgids = [sgids_by_name[sg] for sg in sgs]
res = conn.run_instances(
'ami', security_group_ids=sgids, subnet_id=subnet.id,
private_ip_address=address)
inst = res.instances[0]
inst.add_tag('Name', name)
l.info("Instance {} has id {} in subnet {} with security groups {}".format(
name, inst.id, subnet.id, ', '.join(sgids)))
to_port=22,
cidr_ip='0.0.0.0/0')
sg_public.authorize(ip_protocol='tcp',
from_port=80,
to_port=80,
cidr_ip='0.0.0.0/0')
sg_private = conn.create_security_group('SG for Private Sbunet',
'Private group for VPC via Python',
vpc.id)
sg_private.authorize(ip_protocol='tcp',
from_port=22,
to_port=22,
cidr_ip='0.0.0.0/0')
vpc.add_tag("Name", "Python VPC")
pubsub.add_tag("Name", "Public Subnet by Python")
prvsub.add_tag("Name", "Private Subnet by Python")
igw.add_tag("Name", "Python IGW")
rt.add_tag("Name", "Python Route Table")
sg_public.add_tag("Name", "Public Subnet SG")
sg_private.add_tag("Name", "Private Subnet SG")
print "Security Groups Done!"
print 'VPC Complete!'
#=======================================================================
#Time to create the NAT instance in the public space
ec2 = boto.ec2.connect_to_region('us-east-1')
reservation = ec2.run_instances('ami-184dc970',
key_name='east_keypair',
instance_type='t2.micro',
def __create_vpc (self):
vpc = self.vpc_conn.create_vpc (cidr_block = "172.16.0.0/16");
vpc.add_tag("Name", self.conf["name"] + "_VPC");
return vpc;
def one_time_provision(secrets, path, region, availability_zones, key_name = None):
# 1 region
# 2 VPCs, prod and nonprod
# 3 AZs in each VPC
# 6 subnets per AZ
# 6*3 = 18 subnets
# 32 /26 subnets in the VPC's /21
from netaddr import IPNetwork # sudo pip install netaddr
subnet_size = 24
desired_security_groups = json.load(open('config/security_groups.json', 'r'))
import boto.vpc
import boto.ec2
import boto.exception
vpcs = {}
vpcs[region] = {}
conn_vpc = boto.vpc.connect_to_region(region)
conn_ec2 = boto.ec2.connect_to_region(region)
desired_vpcs = {'us-west-2':
[{'Name':'identity-dev',
'App':'identity',
'Env':'dev',
'cidr':'10.148.24.0/21'},
{'Name':'identity-prod',
'App':'identity',
'Env':'prod',
'cidr':'10.148.32.0/21',
'vpn_target': '63.245.216.58'}
],
'us-east-1':
[{'Name':'identity-dev',
'App':'identity',
'Env':'dev',
'cidr':'10.146.24.0/21'},
{'Name':'identity-prod',
'App':'identity',
'Env':'prod',
'cidr':'10.146.32.0/21',
'vpn_target': '63.245.216.58'}
]
}
asn_map = {'us-west-2': 65148,
'us-east-1': 65146}
ami_map = json.load(open('config/ami_map.json', 'r'))
if not key_name:
key_name = 'svcops-sl62-base-key-%s' % region
for desired_vpc in desired_vpcs[region]:
environment=desired_vpc['Name']
existing_vpcs = conn_vpc.get_all_vpcs()
if desired_vpc['Name'] in [x.tags['Name'] for x in existing_vpcs if 'Name' in x.tags]:
logging.debug('skipping creation of vpc %s since it already exists' % desired_vpc['Name'])
#continue
vpcs[region][environment] = {}
# Create VPCs
vpcs[region][environment]['vpc'] = [x.tags['Name'] for x in existing_vpcs if 'Name' in x.tags]
vpcs[region][environment]['vpc'] = conn_vpc.create_vpc(desired_vpc['cidr'])
vpc = vpcs[region][environment]['vpc']
logging.debug('created vpc %s with id %s and ip range %s' % (environment, vpc.id, vpc.cidr_block))
if vpc.state != 'available':
time.sleep(1)
vpc = conn_vpc.get_all_vpcs([vpc.id])[0]
vpc.add_tag('Name', environment)
vpc.add_tag('App', desired_vpc['App'])
vpc.add_tag('Env', desired_vpc['Env'])
logging.debug('tagged vpc %s with %s' % (vpc.cidr_block, environment))
# Create all security groups
vpcs[region][environment]['security-groups'] = {}
for security_group_definition in desired_security_groups:
security_group_name = environment + '-' + security_group_definition[0]
vpcs[region][environment]['security-groups'][security_group_name] = conn_ec2.create_security_group(
security_group_name,
security_group_name,
vpc.id)
security_group = vpcs[region][environment]['security-groups'][security_group_name]
# This loop is to workaround the race condition between creating the security group
# and the security group being available for use
attempts=0
while True:
try:
# This is to workaround the fact that AWS only returns the groupId when CreateSecurityGroup is called
# instead of the entire object
attempts += 1
security_group = conn_ec2.get_all_security_groups(group_ids=[security_group.id])[0]
break
except boto.exception.EC2ResponseError:
time.sleep(1)
if attempts > 5:
raise
security_group.add_tag('Name', security_group_name)
security_group.add_tag('App', desired_vpc['App'])
security_group.add_tag('Env', desired_vpc['Env'])
# Delete the default egress authorization
conn_ec2.revoke_security_group_egress(group_id=security_group.id, ip_protocol=-1, cidr_ip='0.0.0.0/0')
# And create an internal egress authorization
conn_ec2.authorize_security_group_egress(group_id=security_group.id, ip_protocol=-1, cidr_ip=desired_vpc['cidr'])
logging.debug('created security group %s in VPC %s' % (security_group.name, security_group.vpc_id))
for security_group_definition_rule in security_group_definition[1]:
rule = security_group_definition_rule.copy()
rule['group_id'] = security_group.id
# Handle rules where we set the cidr_ip to "vpc"
if 'cidr_ip' in rule and rule['cidr_ip'] == 'vpc':
rule['cidr_ip'] = desired_vpc['cidr']
if 'direction' not in rule:
rule['direction'] = 'ingress'
# This is to deal with the fact that ingress and egress authorizations work differently
if 'src_security_group_name' in rule:
rule['src_security_group_name'] = environment + '-' + rule['src_security_group_name']
src_group = vpcs[region][environment]['security-groups'][rule['src_security_group_name']]
if rule['direction'] == 'egress':
# egress
rule['src_group_id'] = src_group.id
else:
# ingress
rule['src_security_group_owner_id'] = src_group.owner_id
rule['src_security_group_group_id'] = src_group.id
del rule['src_security_group_name']
if rule['direction'] == 'ingress':
del rule['direction']
if not conn_ec2.authorize_security_group(**rule):
logging.error('failed to add ingress rule %s to security group %s' % (rule,security_group_name))
else:
del rule['direction']
if not conn_ec2.authorize_security_group_egress(**rule):
logging.error('failed to add egress rule %s to security group %s' % (rule,security_group_name))
logging.debug('added rule %s to security group %s' % (rule, security_group_name))
# Create internet gateway (a separate one is required for each VPC)
vpcs[region][environment]['internet_gateway'] = conn_vpc.create_internet_gateway()
# not testing to validate that the ig exists
internet_gateway = vpcs[region][environment]['internet_gateway']
if not conn_vpc.attach_internet_gateway(internet_gateway.id, vpc.id):
logging.error('failed to attach internet gateway %s to vpc %s' % (internet_gateway.id, vpc.id))
# TODO : tag the internet_gateway
# Create VPN to PHX1
if 'vpn_target' in desired_vpc:
customer_gateway = conn_vpc.create_customer_gateway('ipsec.1', desired_vpc['vpn_target'], asn_map[region])
vpn_gateway = conn_vpc.create_vpn_gateway('ipsec.1')
vpn_connection = conn_vpc.create_vpn_connection('ipsec.1', customer_gateway.id, vpn_gateway.id)
# routing : dynamic
customer_gateway_configuration = vpn_connection.customer_gateway_configuration
# TODO set the route table to allow route propoation from the VGW
vpn_gateway_attachment = conn_vpc.attach_vpn_gateway(vpn_gateway.id, vpc.id)
#conn_vpc.create_vpn_connection_route(destination_cidr_block, vpn_connection_id)
#conn_vpc.create_route(route_table_id, destination_cidr_block, gateway_id=None, instance_id=None)
# Create subnets
vpcs[region][environment]['availability_zones'] = {}
ip = IPNetwork(vpc.cidr_block)
available_subnets = ip.subnet(subnet_size)
for availability_zone in [region + x for x in availability_zones]:
vpcs[region][environment]['availability_zones'][availability_zone] = {}
vpcs[region][environment]['availability_zones'][availability_zone]['subnets'] = {}
for subnet_type in ['public', 'private']:
vpcs[region][environment]['availability_zones'][availability_zone]['subnets'][subnet_type] = conn_vpc.create_subnet(
vpc.id,
available_subnets.next(),
availability_zone=availability_zone)
subnet = vpcs[region][environment]['availability_zones'][availability_zone]['subnets'][subnet_type]
if subnet.state != 'available':
time.sleep(1)
subnet = conn_vpc.get_all_subnets([subnet.id])[0]
subnet.add_tag('Name', environment + '-' + subnet_type + '-' + availability_zone)
subnet.add_tag('App', desired_vpc['App'])
subnet.add_tag('Env', desired_vpc['Env'])
logging.debug('created %s subnet %s in VPC %s in AZ %s' % (subnet_type, subnet.cidr_block, subnet.vpc_id, subnet.availability_zone))
# http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-ItemType-SubnetType.html
# Spin up a NAT instance
# We'll just put it in the first availability zone, whatever that is
reservation = conn_ec2.run_instances(image_id = ami_map['ami-vpc-nat-1.0.0-beta.i386-ebs'][region],
key_name = key_name,
security_group_ids = [vpcs[region][environment]['security-groups'][environment + '-' + 'natsg'].id],
instance_type = 't1.micro',
subnet_id = vpcs[region][environment]['availability_zones'][region + availability_zones[0]]['subnets']['public'].id)
vpcs[region][environment]['nat_instance'] = {}
# Loop to wait for the instance to spin up
attempts = 0
sleep_time = 10
max_attempts = 600
while True:
try:
nat_instance_state = conn_ec2.get_all_instances([reservation.instances[0].id])[0].instances[0].state
except (boto.exception.EC2ResponseError, IndexError):
nat_instance_state = False
attempts += 1
if nat_instance_state != 'running':
logging.debug('waiting 10 seconds while instance %s enters a "running" state in order to assign it\'s EIP' % reservation.instances[0].id)
time.sleep(sleep_time)
if attempts > max_attempts:
logging.error('after 10 minutes instance %s remains in a state other than "running". continuing with EIP association which will fail' % reservation.instances[0].id)
else:
break
vpcs[region][environment]['nat_instance']['instance'] = conn_ec2.get_all_instances([reservation.instances[0].id])[0].instances[0]
nat_instance = vpcs[region][environment]['nat_instance']['instance']
nat_instance.add_tag('Name', environment + '-nat_instance')
nat_instance.add_tag('App', desired_vpc['App'])
nat_instance.add_tag('Env', desired_vpc['Env'])
# Get and assign EIP to NAT instance
vpcs[region][environment]['nat_instance']['address'] = conn_ec2.allocate_address('vpc')
address = vpcs[region][environment]['nat_instance']['address']
# TODO : tag the address
conn_ec2.associate_address(instance_id=nat_instance.id,
public_ip=None,
allocation_id=address.allocation_id
)
# Not using address.associate while waiting for merge of https://github.com/boto/boto/pull/1310
# address.associate(nat_instance.id)
# Disabling Source/Destination Checks
if not conn_ec2.modify_instance_attribute(nat_instance.id, 'sourceDestCheck', False):
logging.error('failed to disable Source/Desk Checks on nat_instance %s' % nat_instance.id)
# Route tables
vpcs[region][environment]['route_tables'] = {}
# Create public route table
vpcs[region][environment]['route_tables']['public'] = conn_vpc.create_route_table(vpc.id)
route_table = vpcs[region][environment]['route_tables']['public']
if not conn_vpc.create_route(route_table_id = route_table.id,
destination_cidr_block = '0.0.0.0/0',
gateway_id = internet_gateway.id):
logging.error('failed to add route sending 0.0.0.0/0 traffic to internet gateway %s in route table %s' % (internet_gateway.id, route_table.id))
# TODO : tag the route_table
# Associate public subnets with route table
for availability_zone in [region + x for x in availability_zones]:
subnet = vpcs[region][environment]['availability_zones'][availability_zone]['subnets']['public']
route_table_association = conn_vpc.associate_route_table(route_table.id,
subnet.id)
logging.debug('associated subnet %s with route table %s' % (subnet.id, route_table.id))
# Create private route table
vpcs[region][environment]['route_tables']['private'] = conn_vpc.create_route_table(vpc.id)
route_table = vpcs[region][environment]['route_tables']['private']
# TODO : tag the route_table
# TODO add a route for the PHX1 DB 10.18.20.21/32
# Associate private subnets with route table
for availability_zone in [region + x for x in availability_zones]:
subnet = vpcs[region][environment]['availability_zones'][availability_zone]['subnets']['private']
route_table_association = conn_vpc.associate_route_table(route_table.id,
subnet.id)
logging.debug('associated subnet %s with route table %s' % (subnet.id, route_table.id))
# Set Instance NAT as gateway route for private route table
if not conn_vpc.create_route(route_table_id = route_table.id,
destination_cidr_block = '0.0.0.0/0',
instance_id = nat_instance.id):
logging.error('failed to add route sending 0.0.0.0/0 traffic to nat instance %s in route table %s' % (nat_instance.id, route_table.id))
if 'vpn_target' in desired_vpc:
logging.info('Customer Gateway Configuration = "\n%s\n"' % customer_gateway_configuration)
logging.debug('vpc created')
#pickle.dump(vpcs[region][environment], open(pkl_filename, 'wb'))
#logging.debug('pickled vpc to %s' % pkl_filename)
return vpcs
#public route to IGW
conn.create_route(rt.id, '0.0.0.0/0',igw.id)
#setup Security Group
sg_public = conn.create_security_group('SG for Public Sbunet','Public group for VPC via Python',vpc.id)
sg_public.authorize(ip_protocol='tcp', from_port=22,to_port=22,cidr_ip='0.0.0.0/0')
sg_public.authorize(ip_protocol='tcp', from_port=80,to_port=80,cidr_ip='0.0.0.0/0')
sg_private = conn.create_security_group('SG for Private Sbunet','Private group for VPC via Python',vpc.id)
sg_private.authorize(ip_protocol='tcp', from_port=22,to_port=22,cidr_ip='0.0.0.0/0')
vpc.add_tag("Name", "Python VPC")
pubsub.add_tag("Name","Public Subnet by Python")
prvsub.add_tag("Name","Private Subnet by Python")
igw.add_tag("Name","Python IGW")
rt.add_tag("Name","Python Route Table")
sg_public.add_tag ("Name","Public Subnet SG")
sg_private.add_tag("Name","Private Subnet SG")
print "Security Groups Done!"
print 'VPC Complete!'
#=======================================================================
#Time to create the NAT instance in the public space
ec2 = boto.ec2.connect_to_region('us-east-1')
reservation = ec2.run_instances(
'ami-184dc970',
key_name='east_keypair',
def __create_vpc(self):
vpc = self.vpc_conn.create_vpc(cidr_block="172.16.0.0/16")
vpc.add_tag("Name", self.conf["name"] + "_VPC")
return vpc
def build_private_public_vpc(cidr, key_user, domain_name, aws_id=None or AWS_ID, aws_key=None or AWS_KEY,
region=None or REGION):
"""
Create a VPC with one private and one public subnet, in 2 different availability zones chosed at random
:param cidr: The CIDR for your VPC; min /16, max /28
:param key_user: The name of the AWS PEM key user
:param domain_name: the domain name for your internal DNS resolution in the DHCP option
:param aws_id: Amazon Access Key ID
:param aws_key: Amazon Secret Access Key
:param region: Target region for the VPC
:return: Nothing
"""
vpc_conn = boto.vpc.connect_to_region(region_name=region, aws_access_key_id=aws_id, aws_secret_access_key=aws_key)
ec2_conn = boto.ec2.connect_to_region(region_name=region, aws_access_key_id=aws_id, aws_secret_access_key=aws_key)
av_zones = ec2_conn.get_all_zones()
# Check if a VPC with that CIDR already exists
vpcs = test_vpc_cidr(cidr=cidr, vpc_conn=vpc_conn)
if vpcs:
print(red('You have a VPC withing the desired subnet already, aborting...'))
sys.exit(1)
else:
# Create the VPC
vpc = vpc_conn.create_vpc(cidr_block=cidr)
vpc.add_tag('Name', 'Public/Private VPC')
vpc_conn.modify_vpc_attribute(vpc_id=vpc.id, enable_dns_support=True)
vpc_conn.modify_vpc_attribute(vpc_id=vpc.id, enable_dns_hostnames=True)
print('VPC {} in {} ({}) created'.format(vpc.id, AWS_REGIONS[vpc.region.name], vpc.region.name))
# Create the DHCP Option
dhcp_options = vpc_conn.create_dhcp_options(domain_name=domain_name, domain_name_servers=['AmazonProvidedDNS'],
ntp_servers=[
gethostbyname('0.amazon.pool.ntp.org'),
gethostbyname('1.amazon.pool.ntp.org'),
gethostbyname('2.amazon.pool.ntp.org'),
gethostbyname('3.amazon.pool.ntp.org')
])
dhcp_options.add_tag('Name', domain_name + ' internal')
vpc_conn.associate_dhcp_options(dhcp_options_id=dhcp_options.id, vpc_id=vpc.id)
print('DHCP Options {} created'.format(dhcp_options.id))
# Create an Internet Gateway
internet_gateway = vpc_conn.create_internet_gateway()
print('Internet Gateway {} created'.format(internet_gateway.id))
vpc_conn.attach_internet_gateway(internet_gateway.id, vpc_id=vpc.id)
internet_gateway.add_tag("Name", "Internet Gateway " + vpc.tags['Name'])
# Divide the VPC in subnet_dict
subnetting = calculate_public_private_cidr(vpc_cidr=cidr, av_zones=av_zones)
subnet_dict = {}
for key, item in subnetting.iteritems():
subnet_dict[key] = vpc_conn.create_subnet(vpc_id=vpc.id, cidr_block=item['Network'].cidr,
availability_zone=item['Zone'].name)
subnet_dict[key].add_tag('Name', key + '-' + item['Zone'].name)
# Tag the Main Route Table
vpc_filter = {'vpc-id': vpc.id}
main_route_table = vpc_conn.get_all_route_tables(filters=vpc_filter)[0]
main_route_table.add_tag('Name', 'Main Route Table')
# Create the Public Route Table
public_route_table = vpc_conn.create_route_table(vpc_id=vpc.id)
print('Public Route Table {} created'.format(public_route_table.id))
public_route_table.add_tag('Name', 'Public Route Table')
vpc_conn.create_route(route_table_id=public_route_table.id, destination_cidr_block='0.0.0.0/0',
gateway_id=internet_gateway.id)
print('Public Route created')
# Associate the public subnet
vpc_conn.associate_route_table(route_table_id=public_route_table.id, subnet_id=subnet_dict['Public'].id)
# Create a SSH key for the region
key_name = key_user + '-' + vpc.region.name
keys = [x for x in ec2_conn.get_all_key_pairs() if key_name in x.name]
if keys:
print('SSH key {} already exists'.format(key_name))
key = keys[0]
else:
key = ec2_conn.create_key_pair(key_name=key_name)
print('SSH key {} created'.format(key.name))
key.save(DEFAULT_SSH_DIR)
print('SSH key downloaded in {}'.format(DEFAULT_SSH_DIR))
# Create a NAT instance
nat_instance = spin_nat(subnet_id=subnet_dict['Public'].id, key_name=key.name, env_tag='prd')
print('NAT instance {} created'.format(nat_instance.id))
# Create Private Route Table
private_route_table = vpc_conn.create_route_table(vpc_id=vpc.id)
print('Private Route Table {} created'.format(private_route_table.id))
private_route_table.add_tag('Name', 'Private Route Table')
vpc_conn.create_route(route_table_id=private_route_table.id, destination_cidr_block='0.0.0.0/0',
instance_id=nat_instance.id)
print('Private Route created')
# Associate the private subnet
vpc_conn.associate_route_table(route_table_id=private_route_table.id, subnet_id=subnet_dict['Private'].id)
print(green('VPC succesfully created!'))
print(red('Remember to create manually the Private Hosted Zone {}'.format(domain_name)))
