def delete_network():
existing_vpcs = u.get_vpc_dict()
if VPC_NAME in existing_vpcs:
vpc = ec2.Vpc(existing_vpcs[VPC_NAME].id)
print("Deleting VPC %s (%s) subresources:" % (VPC_NAME, vpc.id))
for subnet in vpc.subnets.all():
try:
sys.stdout.write("Deleting subnet %s ... " % subnet.id)
sys.stdout.write(response_type(subnet.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
for gateway in vpc.internet_gateways.all():
sys.stdout.write("Deleting gateway %s ... " % gateway.id)
# note: if instances are using VPC, this fails with
# botocore.exceptions.ClientError: An error occurred (DependencyViolation) when calling the DetachInternetGateway operation: Network vpc-ca4abab3 has some mapped public address(es). Please unmap those public address(es) before detaching the gateway.
sys.stdout.write('detached ... ' if u.is_good_response(
gateway.detach_from_vpc(VpcId=vpc.id)) else ' detach_failed ')
sys.stdout.write('deleted ' if u.is_good_response(gateway.delete(
)) else ' delete_failed ')
sys.stdout.write('\n')
def desc():
return "%s (%s)" % (route_table.id, u.get_name(route_table.tags))
for route_table in vpc.route_tables.all():
sys.stdout.write(f"Deleting route table {desc()} ... ")
try:
sys.stdout.write(response_type(route_table.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
def desc():
return "%s (%s, %s)" % (security_group.id,
u.get_name(security_group.tags),
security_group.group_name)
for security_group in vpc.security_groups.all():
# default group is undeletable, skip
if security_group.group_name == 'default':
continue
sys.stdout.write('Deleting security group %s ... ' % (desc()))
try:
sys.stdout.write(response_type(security_group.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
sys.stdout.write("Deleting VPC %s ... " % vpc.id)
try:
sys.stdout.write(response_type(vpc.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
def network_setup() -> Tuple[Any, Any]:
"""Creates VPC if it doesn't already exists, configures it for public
internet access, returns vpc, subnet, security_group"""
# from https://gist.github.com/nguyendv/8cfd92fc8ed32ebb78e366f44c2daea6
ec2 = u.get_ec2_resource()
client = u.get_ec2_client()
existing_vpcs = u.get_vpc_dict()
zones = u.get_zones()
# create VPC from scratch. Remove this if default VPC works well enough.
create_non_default_vpc = False
if create_non_default_vpc:
vpc_name = u.get_vpc_name()
if u.get_vpc_name() in existing_vpcs:
print("Reusing VPC " + vpc_name)
vpc = existing_vpcs[vpc_name]
subnets = list(vpc.subnets.all())
assert len(subnets) == len(
zones
), "Has %s subnets, but %s zones, something went wrong during resource creation, try delete_resources.py/create_resources.py" % (
len(subnets), len(zones))
else:
print("Creating VPC " + vpc_name)
vpc = ec2.create_vpc(CidrBlock='192.168.0.0/16')
# enable DNS on the VPC
response = vpc.modify_attribute(EnableDnsHostnames={"Value": True})
assert u.is_good_response(response)
response = vpc.modify_attribute(EnableDnsSupport={"Value": True})
assert u.is_good_response(response)
vpc.create_tags(Tags=u.create_name_tags(vpc_name))
vpc.wait_until_available()
gateways = u.get_gateway_dict(vpc)
gateway_name = u.get_gateway_name()
if gateway_name in gateways:
print("Reusing gateways " + gateway_name)
else:
print("Creating internet gateway " + gateway_name)
ig = ec2.create_internet_gateway()
ig.attach_to_vpc(VpcId=vpc.id)
ig.create_tags(Tags=u.create_name_tags(gateway_name))
# check that attachment succeeded
attach_state = u.extract_attr_for_match(ig.attachments,
State=-1,
VpcId=vpc.id)
assert attach_state == 'available', "vpc %s is in state %s" % (
vpc.id, attach_state)
route_table = vpc.create_route_table()
route_table_name = u.get_route_table_name()
route_table.create_tags(Tags=u.create_name_tags(route_table_name))
dest_cidr = '0.0.0.0/0'
route_table.create_route(DestinationCidrBlock=dest_cidr,
GatewayId=ig.id)
# check success
for route in route_table.routes:
# result looks like this
# ec2.Route(route_table_id='rtb-a8b438cf',
# destination_cidr_block='0.0.0.0/0')
if route.destination_cidr_block == dest_cidr:
break
else:
# sometimes get
# AssertionError: Route for 0.0.0.0/0 not found in [ec2.Route(route_table_id='rtb-cd9153b0', destination_cidr_block='192.168.0.0/16')]
# TODO: add a wait/retry?
assert False, "Route for %s not found in %s" % (
dest_cidr, route_table.routes)
assert len(zones) <= 16 # for cidr/20 to fit into cidr/16
ip = 0
for zone in zones:
cidr_block = '192.168.%d.0/20' % (ip, )
ip += 16
print("Creating subnet %s in zone %s" % (cidr_block, zone))
subnet = vpc.create_subnet(CidrBlock=cidr_block,
AvailabilityZone=zone)
subnet.create_tags(Tags=[{
'Key': 'Name',
'Value': f'{vpc_name}-subnet'
}, {
'Key': 'Region',
'Value': zone
}])
response = client.modify_subnet_attribute(
MapPublicIpOnLaunch={'Value': True}, SubnetId=subnet.id)
assert u.is_good_response(response)
u.wait_until_available(subnet)
assert subnet.map_public_ip_on_launch, "Subnet doesn't enable public IP by default, why?"
route_table.associate_with_subnet(SubnetId=subnet.id)
# Setup security group for non-default VPC
# existing_security_groups = u.get_security_group_dict()
# security_group_nd_name = u.get_security_group_nd_name()
# if security_group_nd_name in existing_security_groups:
# print("Reusing non-default security group " + security_group_nd_name)
# security_group_nd = existing_security_groups[security_group_nd_name]
# assert security_group_nd.vpc_id == vpc.id, f"Found non-default security group {security_group_nd} " \
# f"attached to {security_group_nd.vpc_id} but expected {vpc.id}"
# else:
# security_group_nd = create_security_group(security_group_nd_name, vpc.id)
# Setup things on default VPC for zone-agnostic launching
vpc = u.get_default_vpc()
if not vpc:
util.log(f"Creating default VPC for region {u.get_region()}")
client.create_default_vpc()
vpc = u.get_default_vpc()
assert vpc, "Could not create default VPC?"
existing_security_groups = u.get_security_group_dict()
security_group_name = u.get_security_group_name()
if security_group_name in existing_security_groups:
print("Reusing security group " + security_group_name)
security_group = existing_security_groups[security_group_name]
assert security_group.vpc_id == vpc.id, f"Found security group {security_group} " \
f"attached to {security_group.vpc_id} but expected {vpc.id}"
else:
security_group = create_security_group(security_group_name, vpc.id)
# Uncomment the following when setting up two VPC's
# security_group = create_security_group(security_group_name, vpc.id, security_group_nd)
return vpc, security_group
def delete_network():
if u.get_region() == 'us-east-1':
util.log(
"(Internal safety switch. Not deleting resources in us-east-1, remove this line if you are really sure"
)
return
def delete_vpc(vpc, partial=True):
"""Deletes VPC + all resources, if "partial" set to True, only deletes associated security groups
"""
print("Deleting VPC %s (%s) subresources:" % (VPC_NAME, vpc.id))
# don't modify default VPC
if not partial:
for subnet in vpc.subnets.all():
try:
sys.stdout.write("Deleting subnet %s ... " % subnet.id)
sys.stdout.write(response_type(subnet.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
for gateway in vpc.internet_gateways.all():
sys.stdout.write("Deleting gateway %s ... " % gateway.id)
# note: if instances are using VPC, this fails with
# botocore.exceptions.ClientError: An error occurred (DependencyViolation) when calling the DetachInternetGateway operation: Network vpc-ca4abab3 has some mapped public address(es). Please unmap those public address(es) before detaching the gateway.
sys.stdout.write('detached ... ' if u.is_good_response(
gateway.detach_from_vpc(
VpcId=vpc.id)) else ' detach_failed ')
sys.stdout.write('deleted ' if u.is_good_response(
gateway.delete()) else ' delete_failed ')
sys.stdout.write('\n')
def desc():
return "%s (%s)" % (route_table.id, u.get_name(
route_table.tags))
for route_table in vpc.route_tables.all():
sys.stdout.write(f"Deleting route table {desc()} ... ")
try:
sys.stdout.write(
response_type(route_table.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
else:
util.log(
f"vpc {vpc.id} is a default VPC, only doing partial deletion")
def desc():
return "%s (%s, %s)" % (security_group.id,
u.get_name(security_group.tags),
security_group.group_name)
ncluster_security_groups = u.get_security_group_names()
for security_group in vpc.security_groups.all():
# default group is undeletable, skip
if security_group.group_name == 'default':
continue
# don't delete groups created outside of ncluster framework
if security_group.group_name not in ncluster_security_groups:
continue
sys.stdout.write('Deleting security group %s ... ' % (desc()))
try:
sys.stdout.write(response_type(security_group.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
if not partial:
sys.stdout.write("Deleting VPC %s ... " % vpc.id)
try:
sys.stdout.write(response_type(vpc.delete()) + '\n')
except Exception as e:
sys.stdout.write('failed\n')
util.log_error(str(e) + '\n')
existing_vpcs = u.get_vpc_dict()
if VPC_NAME in existing_vpcs:
# delete ncluster VPC
delete_vpc(ec2.Vpc(existing_vpcs[VPC_NAME].id), partial=False)
# delete ncluster resources on default VPC (partial=True)
delete_vpc(u.get_default_vpc())
def network_setup():
"""Creates VPC if it doesn't already exists, configures it for public
internet access, returns vpc, subnet, security_group"""
# from https://gist.github.com/nguyendv/8cfd92fc8ed32ebb78e366f44c2daea6
ec2 = u.get_ec2_resource()
client = u.get_ec2_client()
existing_vpcs = u.get_vpc_dict()
zones = u.get_zones()
# create VPC from scratch. Remove this if default VPC works well enough.
vpc_name = u.get_vpc_name()
if u.get_vpc_name() in existing_vpcs:
print("Reusing VPC " + vpc_name)
vpc = existing_vpcs[vpc_name]
subnets = list(vpc.subnets.all())
assert len(subnets) == len(
zones
), "Has %s subnets, but %s zones, something went wrong during resource creation, try delete_resources.py/create_resources.py" % (
len(subnets), len(zones))
else:
print("Creating VPC " + vpc_name)
vpc = ec2.create_vpc(CidrBlock='192.168.0.0/16')
# enable DNS on the VPC
response = vpc.modify_attribute(EnableDnsHostnames={"Value": True})
assert u.is_good_response(response)
response = vpc.modify_attribute(EnableDnsSupport={"Value": True})
assert u.is_good_response(response)
vpc.create_tags(Tags=u.create_name_tags(vpc_name))
vpc.wait_until_available()
gateways = u.get_gateway_dict(vpc)
gateway_name = u.get_gateway_name()
if gateway_name in gateways:
print("Reusing gateways " + gateway_name)
else:
print("Creating internet gateway " + gateway_name)
ig = ec2.create_internet_gateway()
ig.attach_to_vpc(VpcId=vpc.id)
ig.create_tags(Tags=u.create_name_tags(gateway_name))
# check that attachment succeeded
attach_state = u.extract_attr_for_match(ig.attachments,
State=-1,
VpcId=vpc.id)
assert attach_state == 'available', "vpc %s is in state %s" % (
vpc.id, attach_state)
route_table = vpc.create_route_table()
route_table_name = u.get_route_table_name()
route_table.create_tags(Tags=u.create_name_tags(route_table_name))
dest_cidr = '0.0.0.0/0'
route_table.create_route(DestinationCidrBlock=dest_cidr,
GatewayId=ig.id)
# check success
for route in route_table.routes:
# result looks like this
# ec2.Route(route_table_id='rtb-a8b438cf',
# destination_cidr_block='0.0.0.0/0')
if route.destination_cidr_block == dest_cidr:
break
else:
# sometimes get
# AssertionError: Route for 0.0.0.0/0 not found in [ec2.Route(route_table_id='rtb-cd9153b0', destination_cidr_block='192.168.0.0/16')]
# TODO: add a wait/retry?
assert False, "Route for %s not found in %s" % (dest_cidr,
route_table.routes)
assert len(zones) <= 16 # for cidr/20 to fit into cidr/16
ip = 0
for zone in zones:
cidr_block = '192.168.%d.0/20' % (ip, )
ip += 16
print("Creating subnet %s in zone %s" % (cidr_block, zone))
subnet = vpc.create_subnet(CidrBlock=cidr_block,
AvailabilityZone=zone)
subnet.create_tags(Tags=[{
'Key': 'Name',
'Value': f'{vpc_name}-subnet'
}, {
'Key': 'Region',
'Value': zone
}])
response = client.modify_subnet_attribute(
MapPublicIpOnLaunch={'Value': True}, SubnetId=subnet.id)
assert u.is_good_response(response)
u.wait_until_available(subnet)
assert subnet.map_public_ip_on_launch, "Subnet doesn't enable public IP by default, why?"
route_table.associate_with_subnet(SubnetId=subnet.id)
# Use default VPC from now on
vpc = u.get_default_vpc()
if not vpc:
util.log(f"Creating default VPC for region {u.get_region()}")
client.create_default_vpc()
vpc = u.get_default_vpc()
assert vpc, "Could not create default VPC?"
existing_security_groups = u.get_security_group_dict()
security_group_name = u.get_security_group_name()
if security_group_name in existing_security_groups:
print("Reusing security group " + security_group_name)
security_group = existing_security_groups[security_group_name]
assert security_group.vpc_id == vpc.id, f"Found security group {security_group} " \
f"attached to {security_group.vpc_id} but expected {vpc.id}"
else:
print("Creating security group " + security_group_name)
security_group = ec2.create_security_group(
GroupName=security_group_name,
Description=security_group_name,
VpcId=vpc.id)
security_group.create_tags(
Tags=u.create_name_tags(security_group_name))
# allow ICMP access for public ping
security_group.authorize_ingress(CidrIp='0.0.0.0/0',
IpProtocol='icmp',
FromPort=-1,
ToPort=-1)
# open public ports
# always include SSH port which is required for basic functionality
assert 22 in PUBLIC_TCP_RANGES, "Must enable SSH access"
for port in PUBLIC_TCP_RANGES:
if util.is_iterable(port):
assert len(port) == 2
from_port, to_port = port
else:
from_port, to_port = port, port
response = security_group.authorize_ingress(IpProtocol="tcp",
CidrIp="0.0.0.0/0",
FromPort=from_port,
ToPort=to_port)
assert u.is_good_response(response)
for port in PUBLIC_UDP_RANGES:
if util.is_iterable(port):
assert len(port) == 2
from_port, to_port = port
else:
from_port, to_port = port, port
response = security_group.authorize_ingress(IpProtocol="udp",
CidrIp="0.0.0.0/0",
FromPort=from_port,
ToPort=to_port)
assert u.is_good_response(response)
# allow ingress within security group
# Authorizing ingress doesn't work with names in a non-default VPC,
# so must use more complicated syntax
# https://github.com/boto/boto3/issues/158
response = {}
for protocol in ['icmp']:
try:
rule = {
'FromPort': -1,
'IpProtocol': protocol,
'IpRanges': [],
'PrefixListIds': [],
'ToPort': -1,
'UserIdGroupPairs': [{
'GroupId': security_group.id
}]
}
response = security_group.authorize_ingress(
IpPermissions=[rule])
except Exception as e:
if response['Error']['Code'] == 'InvalidPermission.Duplicate':
print("Warning, got " + str(e))
else:
assert False, "Failed while authorizing ingress with " + str(
e)
for protocol in ['tcp', 'udp']:
try:
rule = {
'FromPort': 0,
'IpProtocol': protocol,
'IpRanges': [],
'PrefixListIds': [],
'ToPort': 65535,
'UserIdGroupPairs': [{
'GroupId': security_group.id
}]
}
response = security_group.authorize_ingress(
IpPermissions=[rule])
except Exception as e:
if response['Error']['Code'] == 'InvalidPermission.Duplicate':
print("Warning, got " + str(e))
else:
assert False, "Failed while authorizing ingress with " + str(
e)
return vpc, security_group