def open_tcp_port(sec_grp_id, port, ip_address="0.0.0.0/0"):
port = int(port)
create_ec2_client().authorize_security_group_ingress(GroupId=sec_grp_id,
IpProtocol="tcp",
CidrIp=ip_address,
FromPort=port,
ToPort=port)
def get_instances_by_name(instance_name):
""" thank you to https://rob.salmond.ca/filtering-instances-by-name-with-boto3/ for having
sufficient SEO to be a googleable answer on how to even do this.
And then this stack overflow for how to query by instances that are running:
https://stackoverflow.com/questions/37293366/what-is-the-correct-ways-to-write-boto3-filters-to-use-customise-tag-name
"""
reservations = create_ec2_client().describe_instances(Filters=[
{
'Name': 'tag:Name',
'Values': [instance_name]
},
{
'Name': 'instance-state-name',
'Values': ['running']
},
])['Reservations']
# need to concatenate all instances from every "reservation", whatever that is.
instances = []
for reservation in reservations:
instances.extend(reservation['Instances'])
if not instances:
log.error("Could not find any instances matching the name '%s'" %
instance_name)
return instances
def get_most_recent_ubuntu():
""" Unfortunately the different fundamental ec2 server types require a specific image type.
All the ubuntu xenial prefixe matches are defined below, the currently selected is known to
function for T2, M4, and C4 server classes. Other server types may require testing the
different classes
"""
ec2_client = create_ec2_client()
images = ec2_client.describe_images(Filters=[
{
"Name": 'state',
"Values": ['available']
},
{
"Name": 'name',
# "Values": ["ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server*"]}
# "Values": ["ubuntu/images/ubuntu-xenial-16.04-amd64-server*"]}
# "Values": ["ubuntu/images/hvm-instance/ubuntu-xenial-16.04-amd64-server*"]}
"Values":
["ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server*"]
}
])['Images']
# The names are time-sortable, we want the most recent one, it is at the bottom of a sorted list
images.sort(key=lambda x: x['Name'])
log.info("Using AMI '%s'" % images[-1]['Name'])
return images[-1]
def get_most_recent_ubuntu():
""" Unfortunately the different fundamental ec2 server types require a specific image type.
All the ubuntu xenial prefixe matches are defined below, the currently selected is known to
function for T2, M4, and C4 server classes. Other server types may require testing the
different classes, (seems to work with t3, c5, m5)
"""
ec2_client = create_ec2_client()
images = ec2_client.describe_images(Filters=[
{
"Name": 'state',
"Values": ['available']
},
{
"Name": 'name',
"Values":
["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server*"]
},
])['Images']
# The names are time-sortable, we want the most recent one, it is at the bottom of a sorted list
images = [
image for image in images
if "aws-marketplace" not in image["ImageLocation"]
]
images.sort(key=lambda x: x['Name'])
log.info("Using AMI '%s'" % images[-1]['Name'])
return images[-1]
def get_security_group_by_name(sec_grp_name):
try:
return create_ec2_client().describe_security_groups(
GroupNames=[sec_grp_name])['SecurityGroups'][0]
except ClientError as e:
# Boto3 throws unimportable errors.
if "InvalidGroup.NotFound" in str(e):
log.debug(str(e))
raise InvalidSecurityGroupNameException(sec_grp_name)
raise
def get_security_group_by_id(sec_grp_id):
try:
return create_ec2_client().describe_security_groups(
GroupIds=[sec_grp_id])['SecurityGroups'][0]
except ClientError as e:
# Boto3 throws unimportable errors.
if "Invalid id:" in str(e):
log.debug(str(e))
raise InvalidSecurityGroupIdException(sec_grp_id)
raise
def get_instances_by_name(instance_name):
""" thank you to https://rob.salmond.ca/filtering-instances-by-name-with-boto3/ for having
sufficient SEO to be a googleable answer on how to even do this. """
ret = create_ec2_client().describe_instances(
Filters=[{
'Name': 'tag:Name',
'Values': [instance_name]
}])
try:
return ret['Reservations'][0]["Instances"]
except IndexError:
log.warn("could not find any instances matching the name '%s'" %
instance_name)
return []
def terminate_all_processing_servers(eb_environment_name):
ec2_client = create_ec2_client()
worker_ids = [
worker['InstanceId']
for worker in get_worker_instances(eb_environment_name)
]
# don't optimize, we want the log statements
for instance_id in worker_ids:
ec2_client.terminate_instances(InstanceIds=[instance_id])
log.info(f"Terminating worker instance {instance_id}")
manager_info = get_manager_instance_by_eb_environment_name(
eb_environment_name)
if manager_info:
log.info(f"Terminating manager instance {manager_info['InstanceId']}")
ec2_client.terminate_instances(
InstanceIds=[manager_info['InstanceId']])
def create_security_group(group_name,
description,
list_of_dicts_of_ingress_kwargs=None,
list_of_dicts_of_egress_kwargs=None):
"""
mostly kwargs should look like this: ToPort=22, IpProtocol="TCP", SourceSecurityGroupName="thing"
"""
if list_of_dicts_of_ingress_kwargs is None:
list_of_dicts_of_ingress_kwargs = []
if list_of_dicts_of_egress_kwargs is None:
list_of_dicts_of_egress_kwargs = []
if not isinstance(list_of_dicts_of_ingress_kwargs, list):
raise Exception(
"list_of_dicts_of_ingress_kwargs was not a list, it was a %s" %
type(list_of_dicts_of_ingress_kwargs))
if not isinstance(list_of_dicts_of_egress_kwargs, list):
raise Exception(
"list_of_dicts_of_egress_kwargs was not a list, it was a %s" %
type(list_of_dicts_of_egress_kwargs))
ec2_client = create_ec2_client()
ec2_resource = create_ec2_resource()
sec_grp = ec2_client.create_security_group(
VpcId=GLOBAL_CONFIGURATION["VPC_ID"],
GroupName=group_name,
Description=description,
)
sec_grp_resource = ec2_resource.SecurityGroup(sec_grp["GroupId"])
for ingress_params in list_of_dicts_of_ingress_kwargs:
sec_grp_resource.authorize_ingress(**ingress_params)
for egress_params in list_of_dicts_of_egress_kwargs:
sec_grp_resource.authorize_egress(**egress_params)
return get_security_group_by_id(sec_grp["GroupId"])
def get_instance_by_id(instance_id):
ec2_client = create_ec2_client()
return ec2_client.describe_instances(InstanceIds=[instance_id])['Reservations'][0]["Instances"][0]
def create_server(eb_environment_name, aws_server_type, security_groups=None):
ec2_client = create_ec2_client()
if security_groups is None:
security_groups = []
if not isinstance(security_groups, list):
raise Exception("security_groups must be a list, received '%s'" % type(security_groups))
ebs_parameters = {
'DeviceName': '/dev/sda1', # boot drive...
'Ebs': {
'DeleteOnTermination': True,
'VolumeSize': 8,
# gigabytes, No storage is required on any ubuntu machines, 8 is default
'VolumeType': 'gp2'} # SSD...
}
instance = ec2_client.run_instances(
ImageId=get_most_recent_ubuntu()['ImageId'],
MinCount=1,
MaxCount=1,
KeyName=GLOBAL_CONFIGURATION['DEPLOYMENT_KEY_NAME'],
InstanceType=aws_server_type,
SecurityGroupIds=security_groups,
# NetworkInterfaces=[{"DeviceIndex": 0,
# "AssociatePublicIpAddress": True,
# "SubnetId": config.public_subnet_id,
# "Groups": security_groups_list}],
# IamInstanceProfile={"Arn": MANAGER_IAM_ROLE},
BlockDeviceMappings=[ebs_parameters],
Monitoring={'Enabled': False},
InstanceInitiatedShutdownBehavior='stop',
# Placement={'AvailabilityZone': 'string',
# 'Affinity': 'string',
# 'GroupName': 'string',
# 'HostId': 'string',
# 'Tenancy': 'default'|'dedicated'|'host',
# 'SpreadDomain': 'string'
# },
# IamInstanceProfile={'Arn': 'string',
# 'Name': 'string'},
# NetworkInterfaces=[ {
# 'AssociatePublicIpAddress': True|False,
# 'DeleteOnTermination': True|False,
# 'Description': 'string',
# 'DeviceIndex': 123,
# 'Groups': ['string',],
# 'Ipv6AddressCount': 123,
# 'Ipv6Addresses': [ { 'Ipv6Address': 'string' }, ],
# 'NetworkInterfaceId': 'string',
# 'PrivateIpAddress': 'string',
# 'PrivateIpAddresses': [ {'Primary': True|False,
# 'PrivateIpAddress': 'string'},],
# 'SecondaryPrivateIpAddressCount': 123,
# 'SubnetId': 'string'
# }, ],
#
# TagSpecifications=[ {
# 'ResourceType': 'customer-gateway'|'dhcp-options'|'image'|'instance'|'internet-gateway'|'network-acl'|'network-interface'|'reserved-instances'|'route-table'|'snapshot'|'spot-instances-request'|'subnet'|'security-group'|'volume'|'vpc'|'vpn-connection'|'vpn-gateway',
# 'Tags': [ { 'Key': 'string',
# 'Value': 'string'},]
# },
# ]
)["Instances"][0]
instance_id = instance["InstanceId"]
instance_resource = create_ec2_resource().Instance(instance_id)
log.info("Waiting for server %s to show up..." % instance_id)
instance_resource.wait_until_exists()
log.info("Waiting until server %s is up and running (this may take a minute) ..." % instance_id)
instance_resource.wait_until_running()
return get_instance_by_id(instance_id)
