def create_processing_server(eb_environment_name, aws_server_type):
instance_sec_grp_id = get_rds_security_groups_by_eb_name(eb_environment_name)["instance_sec_grp"]['GroupId']
instance_info = create_server(eb_environment_name, aws_server_type,
security_groups=[instance_sec_grp_id])
instance_resource = create_ec2_resource().Instance(instance_info["InstanceId"])
instance_resource.create_tags(Tags=[
{"Key": "Name", "Value": "%s data processing server" % eb_environment_name},
{"Key": "is_processing_worker", "Value": "1"}
])
return instance_info
def add_eb_environment_to_rds_database_security_group(eb_environment_name, eb_sec_grp_id):
""" We need to add the elastic beansalk environment to the security group that we assign to the RDS insnant"""
ingress_params = create_sec_grp_rule_parameters_allowing_traffic_from_another_security_group(
tcp_port=POSTGRES_PORT, sec_grp_id=eb_sec_grp_id
)
_, database_sec_grp_name = get_rds_security_group_names(construct_db_name(eb_environment_name))
db_sec_grp = get_security_group_by_name(database_sec_grp_name)
sec_grp_resource = create_ec2_resource().SecurityGroup(db_sec_grp['GroupId'])
sec_grp_resource.authorize_ingress(**ingress_params)
def create_processing_control_server(eb_environment_name, aws_server_type):
""" The differences between a data processing worker server and a processing controller
server is that the controller needs to allow connections from the processors. """
get_rds_security_groups_by_eb_name(
eb_environment_name)["instance_sec_grp"]['GroupId']
# TODO: functions that terminate all worker and all manager servers for an environment
manager_info = get_manager_instance_by_eb_environment_name(
eb_environment_name)
if manager_info is not None and manager_info['State'][
'Name'] != 'terminated':
if manager_info['InstanceType'] == aws_server_type:
msg = "A manager server, %s, already exists for this environment, and it is of the provided type (%s)." % (
manager_info['InstanceId'], aws_server_type)
else:
msg = "A manager server, %s, already exists for this environment." % manager_info[
'InstanceId']
log.error(msg)
msg = "You must terminate all worker and manager servers before you can create a new manager."
log.error(msg)
sleep(
0.1
) # sometimes log has problems if you don't give it a second, the error messages above are critical
raise Exception(msg)
rabbit_mq_sec_grp_id = get_or_create_rabbit_mq_security_group(
eb_environment_name)['GroupId']
instance_sec_grp_id = get_rds_security_groups_by_eb_name(
eb_environment_name)["instance_sec_grp"]['GroupId']
try:
open_tcp_port(instance_sec_grp_id, 22)
except ClientError:
# we need to open the ssh port for future worker servers, but it blows up with duplicate
# if a user ever creates two managers during the life of the environment.
pass
instance_info = create_server(
eb_environment_name,
aws_server_type,
security_groups=[rabbit_mq_sec_grp_id, instance_sec_grp_id])
instance_resource = create_ec2_resource().Instance(
instance_info["InstanceId"])
instance_resource.create_tags(
Tags=[{
"Key": "Name",
"Value": PROCESSING_MANAGER_NAME % eb_environment_name
}, {
"Key": "is_processing_manager",
"Value": "1"
}])
return instance_info
def create_security_group(group_name,
description,
list_of_dicts_of_ingress_kwargs=None,
list_of_dicts_of_egress_kwargs=None):
"""
mostly kwargs should look like this: ToPort=22, IpProtocol="TCP", SourceSecurityGroupName="thing"
"""
if list_of_dicts_of_ingress_kwargs is None:
list_of_dicts_of_ingress_kwargs = []
if list_of_dicts_of_egress_kwargs is None:
list_of_dicts_of_egress_kwargs = []
if not isinstance(list_of_dicts_of_ingress_kwargs, list):
raise Exception(
"list_of_dicts_of_ingress_kwargs was not a list, it was a %s" %
type(list_of_dicts_of_ingress_kwargs))
if not isinstance(list_of_dicts_of_egress_kwargs, list):
raise Exception(
"list_of_dicts_of_egress_kwargs was not a list, it was a %s" %
type(list_of_dicts_of_egress_kwargs))
ec2_client = create_ec2_client()
ec2_resource = create_ec2_resource()
sec_grp = ec2_client.create_security_group(
VpcId=GLOBAL_CONFIGURATION["VPC_ID"],
GroupName=group_name,
Description=description,
)
sec_grp_resource = ec2_resource.SecurityGroup(sec_grp["GroupId"])
for ingress_params in list_of_dicts_of_ingress_kwargs:
sec_grp_resource.authorize_ingress(**ingress_params)
for egress_params in list_of_dicts_of_egress_kwargs:
sec_grp_resource.authorize_egress(**egress_params)
return get_security_group_by_id(sec_grp["GroupId"])
def create_server(eb_environment_name, aws_server_type, security_groups=None):
ec2_client = create_ec2_client()
if security_groups is None:
security_groups = []
if not isinstance(security_groups, list):
raise Exception("security_groups must be a list, received '%s'" % type(security_groups))
ebs_parameters = {
'DeviceName': '/dev/sda1', # boot drive...
'Ebs': {
'DeleteOnTermination': True,
'VolumeSize': 8,
# gigabytes, No storage is required on any ubuntu machines, 8 is default
'VolumeType': 'gp2'} # SSD...
}
instance = ec2_client.run_instances(
ImageId=get_most_recent_ubuntu()['ImageId'],
MinCount=1,
MaxCount=1,
KeyName=GLOBAL_CONFIGURATION['DEPLOYMENT_KEY_NAME'],
InstanceType=aws_server_type,
SecurityGroupIds=security_groups,
# NetworkInterfaces=[{"DeviceIndex": 0,
# "AssociatePublicIpAddress": True,
# "SubnetId": config.public_subnet_id,
# "Groups": security_groups_list}],
# IamInstanceProfile={"Arn": MANAGER_IAM_ROLE},
BlockDeviceMappings=[ebs_parameters],
Monitoring={'Enabled': False},
InstanceInitiatedShutdownBehavior='stop',
# Placement={'AvailabilityZone': 'string',
# 'Affinity': 'string',
# 'GroupName': 'string',
# 'HostId': 'string',
# 'Tenancy': 'default'|'dedicated'|'host',
# 'SpreadDomain': 'string'
# },
# IamInstanceProfile={'Arn': 'string',
# 'Name': 'string'},
# NetworkInterfaces=[ {
# 'AssociatePublicIpAddress': True|False,
# 'DeleteOnTermination': True|False,
# 'Description': 'string',
# 'DeviceIndex': 123,
# 'Groups': ['string',],
# 'Ipv6AddressCount': 123,
# 'Ipv6Addresses': [ { 'Ipv6Address': 'string' }, ],
# 'NetworkInterfaceId': 'string',
# 'PrivateIpAddress': 'string',
# 'PrivateIpAddresses': [ {'Primary': True|False,
# 'PrivateIpAddress': 'string'},],
# 'SecondaryPrivateIpAddressCount': 123,
# 'SubnetId': 'string'
# }, ],
#
# TagSpecifications=[ {
# 'ResourceType': 'customer-gateway'|'dhcp-options'|'image'|'instance'|'internet-gateway'|'network-acl'|'network-interface'|'reserved-instances'|'route-table'|'snapshot'|'spot-instances-request'|'subnet'|'security-group'|'volume'|'vpc'|'vpn-connection'|'vpn-gateway',
# 'Tags': [ { 'Key': 'string',
# 'Value': 'string'},]
# },
# ]
)["Instances"][0]
instance_id = instance["InstanceId"]
instance_resource = create_ec2_resource().Instance(instance_id)
log.info("Waiting for server %s to show up..." % instance_id)
instance_resource.wait_until_exists()
log.info("Waiting until server %s is up and running (this may take a minute) ..." % instance_id)
instance_resource.wait_until_running()
return get_instance_by_id(instance_id)
