def autoscaling_instance_role(self, role_name, role_data, desired_count):
if role_data.get('autoscaling', False) != True:
# exit early if this instance_role is not autoscaling.
return None
long_role_name = '{}-{}'.format(self.evpc.name, role_name)
ami = self.amis[role_data['ami']][self.evpc.region_name]
key_pair = self.evpc.key_pair.get_key_pair(
role_data.get('key_pair', 'default'))
security_groups = map_filter_false(
self.evpc.get_security_group, role_data.get('security_groups', []))
subnets = map_filter_false(self.evpc.get_subnet,
role_data.get('subnets', []))
block_device_map = get_block_device_map_from_role_config(role_data)
kwargs = {
'LaunchConfigurationName': long_role_name,
'ImageId': ami,
'InstanceType': role_data.get('instance_type'),
'KeyName': key_pair.name,
'SecurityGroups': get_ids(security_groups),
'BlockDeviceMappings': block_device_map,
'UserData': role_data.get('userdata', ''),
}
profile_name = role_data.get('instance_profile_name', None)
if profile_name:
# only add profile if set. Empty string causes error.
kwargs['IamInstanceProfile'] = profile_name
self.wait_for_instance_profile(profile_name)
self.log.emit('creating launch configuration for role: {}'.format(
long_role_name))
self.evpc.autoscaling.create_launch_configuration(**kwargs)
self.log.emit(
'creating autoscaling group for role: {}'.format(long_role_name))
self.evpc.autoscaling.create_auto_scaling_group(
AutoScalingGroupName=long_role_name,
LaunchConfigurationName=long_role_name,
MinSize=desired_count,
MaxSize=desired_count,
DesiredCapacity=desired_count,
VPCZoneIdentifier=','.join(get_ids(subnets)),
Tags=[
{
'Key': 'role',
'Value': role_name,
'PropagateAtLaunch': True,
},
])
def db_instances(self, db_instance_cfg):
"""Build RDS DB Instances."""
existing_rds_names = self.evpc.rds.get_related_db_ids()
for rds_name, db_cfg in db_instance_cfg.items():
if rds_name in existing_rds_names:
msg = "skipping RDS db_instance: {} (it already exists)"
self.log.emit(msg.format(rds_name), "debug")
continue
self.log.emit("creating {} RDS db_instance ...".format(rds_name))
# make list of security group ids.
security_groups = map_filter_false(
self.evpc.get_security_group, db_cfg.get("security_groups", [])
)
sg_ids = get_ids(security_groups)
# make list of subnet ids.
subnets = map_filter_false(self.evpc.get_subnet, db_cfg.get("subnets", []))
sn_ids = get_ids(subnets)
self.evpc.rds.create_db_subnet_group(
DBSubnetGroupName=rds_name,
DBSubnetGroupDescription=db_cfg.get("description", ""),
SubnetIds=sn_ids,
)
self.evpc.rds.create_db_instance(
DBInstanceIdentifier=rds_name,
DBSubnetGroupName=rds_name,
DBName=db_cfg.get("name", rds_name),
VpcSecurityGroupIds=sg_ids,
DBInstanceClass=db_cfg.get("class", "db.t2.medium"),
AllocatedStorage=db_cfg.get("allocated_storage", 100),
Engine=db_cfg.get("engine"),
EngineVersion=db_cfg.get("engine_version", ""),
Iops=db_cfg.get("iops", 0),
MultiAZ=db_cfg.get("multi_az", False),
MasterUsername=db_cfg.get("master_username"),
MasterUserPassword=generate_password(16),
BackupRetentionPeriod=db_cfg.get("backup_retention_period", 0),
StorageType=db_cfg.get("storage_type", "standard"), # 'gp2'
StorageEncrypted=db_cfg.get("storage_encryption", False),
Tags=[{"Key": "vpc_name", "Value": self.evpc.vpc_name}],
)
def db_instances(self, db_instance_cfg):
"""Build RDS DB Instances."""
for rds_name, db_cfg in db_instance_cfg.items():
self.log.emit('creating {} RDS db_instance ...'.format(rds_name))
# make list of security group ids.
security_groups = map_filter_false(
self.evpc.get_security_group,
db_cfg.get('security_groups', []))
sg_ids = get_ids(security_groups)
# make list of subnet ids.
subnets = map_filter_false(self.evpc.get_subnet,
db_cfg.get('subnets', []))
sn_ids = get_ids(subnets)
self.evpc.rds.create_db_subnet_group(
DBSubnetGroupName=rds_name,
DBSubnetGroupDescription=db_cfg.get('description', ''),
SubnetIds=sn_ids,
)
self.evpc.rds.create_db_instance(
DBInstanceIdentifier=rds_name,
DBSubnetGroupName=rds_name,
DBName=db_cfg.get('name', rds_name),
VpcSecurityGroupIds=sg_ids,
DBInstanceClass=db_cfg.get('class', 'db.t2.medium'),
AllocatedStorage=db_cfg.get('allocated_storage', 100),
Engine=db_cfg.get('engine'),
EngineVersion=db_cfg.get('engine_version', ''),
Iops=db_cfg.get('iops', 0),
MultiAZ=db_cfg.get('multi_az', False),
MasterUsername=db_cfg.get('master_username'),
MasterUserPassword=generate_password(16),
BackupRetentionPeriod=db_cfg.get('backup_retention_period', 0),
Tags=[{
'Key': 'vpc_name',
'Value': self.evpc.vpc_name
}],
)
def load_balancers(self, load_balancer_cfg):
"""Build ELB load balancers."""
for lb_name, lb_cfg in load_balancer_cfg.items():
lb_fullname = '{}-{}'.format(self.evpc.name, lb_name)
self.log.emit('creating {} load_balancer ...'.format(lb_fullname))
# make list of security group ids.
security_groups = map_filter_false(
self.evpc.get_security_group,
lb_cfg.get('security_groups', []))
sg_ids = get_ids(security_groups)
# make list of subnet ids.
subnets = map_filter_false(self.evpc.get_subnet,
lb_cfg.get('subnets', []))
sn_ids = get_ids(subnets)
scheme = 'internet-facing'
if lb_cfg.get('internal', False):
scheme = 'internal'
listeners = lb_cfg.get('listeners', [])
self.evpc.elb.create_load_balancer(
LoadBalancerName=lb_fullname,
Subnets=sn_ids,
SecurityGroups=sg_ids,
Scheme=scheme,
Tags=[
{
'Key': 'vpc_name',
'Value': self.evpc.vpc_name
},
{
'Key': 'role',
'Value': lb_cfg['instance_role']
},
],
Listeners=self.evpc.elb.format_listeners(listeners),
)
self.log.emit('created {} load_balancer ...'.format(lb_fullname))
self.log.emit(
'Configure Health Check for {} load_balancer ...'.format(
lb_fullname))
hc_cfg = lb_cfg.get('healthcheck', {})
hc_default_target = 'TCP:{}'.format(listeners[0][1])
self.evpc.elb.configure_health_check(
LoadBalancerName=lb_fullname,
HealthCheck={
'Target': hc_cfg.get('target', hc_default_target),
'Interval': hc_cfg.get('interval', 15),
'Timeout': hc_cfg.get('timeout', 5),
'UnhealthyThreshold': hc_cfg.get('unhealthy_threshold', 4),
'HealthyThreshold': hc_cfg.get('healthy_threshold', 4)
})
self.log.emit(
'Configured Health Check for {} load_balancer ...'.format(
lb_fullname))
asg_name = '{}-{}'.format(self.evpc.name, lb_cfg['instance_role'])
if asg_name in self.evpc.autoscaling.get_related_autoscaling_group_names(
):
self.log.emit(
'attaching {} load balancer to {} autoscaling group ...'.
format(lb_fullname, asg_name))
self.evpc.autoscaling.attach_load_balancers(
AutoScalingGroupName=asg_name,
LoadBalancerNames=[lb_fullname],
)
else:
self.log.emit(
'registering {} role to {} load balancer ...'.format(
lb_cfg['instance_role'], lb_fullname))
self.evpc.elb.register_role_with_load_balancer(
lb_fullname, lb_cfg['instance_role'])
def instance_role(self, role_name, role_data, desired_count):
if role_data.get('autoscaling', False) == True:
# exit early if this instance_role is autoscaling.
return None
self.log.emit('creating role: {}'.format(role_name))
ami = self.amis[role_data['ami']][self.evpc.region_name]
key_pair = self.evpc.key_pair.get_key_pair(
role_data.get('key_pair', 'default'))
security_groups = map_filter_false(
self.evpc.get_security_group, role_data.get('security_groups', []))
subnets = map_filter_false(self.evpc.get_subnet,
role_data.get('subnets', []))
if len(subnets) == 0:
self.log.emit('no subnets found for role: {}'.format(role_name),
'warning')
# exit early.
return None
# sort by subnets by amount of instances, smallest first.
subnets = sorted(
subnets,
key=lambda sn: collection_len(sn.instances),
)
# determine the count of this role's existing instances.
# Note: we look for role in all subnets, not just the listed subnets.
existing_count = len(self.evpc.get_role(role_name))
if existing_count >= desired_count:
# for now we exit early, maybe terminate extras...
msg = 'skipping role: {} (existing_count {} is greater than or equal to {})'
self.log.emit(msg.format(role_name, existing_count, desired_count),
'debug')
return None
# determine count of additional instances needed to reach desired_count.
needed_count = desired_count - existing_count
needed_per_subnet = desired_count / len(subnets)
needed_remainder = desired_count % len(subnets)
block_device_map = get_block_device_map_from_role_config(role_data)
role_instances = []
kwargs = {
'ImageId': ami,
'InstanceType': role_data.get('instance_type'),
'KeyName': key_pair.name,
'SecurityGroupIds': get_ids(security_groups),
'BlockDeviceMappings': block_device_map,
'UserData': role_data.get('userdata', ''),
'IamInstanceProfile': {},
}
profile_name = role_data.get('instance_profile_name', None)
if profile_name:
kwargs['IamInstanceProfile'] = {'Name': profile_name}
for subnet in subnets:
# ensure Run_Instance_Idempotency.html#client-tokens
kwargs['ClientToken'] = str(uuid4())
# figure out how many instances this subnet needs to create ...
existing_in_subnet = len(
self.evpc.get_role(role_name, subnet.instances.all()))
count = needed_per_subnet - existing_in_subnet
if needed_remainder != 0:
needed_remainder -= 1
count += 1
if count == 0:
# skip this subnet, it doesn't need to launch any instances.
continue
subnet_name = make_tag_dict(subnet)['Name']
msg = '{} instances of role {} launching into {} subnet'
self.log.emit(msg.format(count, role_name, subnet_name))
# create a batch of instances in subnet!
kwargs['MinCount'] = kwargs['MaxCount'] = count
instances = self._create_instances(subnet, **kwargs)
# accumulate all new instances into a single list.
role_instances += instances
# add role tag to each instance.
for instance in role_instances:
update_tags(instance, role=role_name)
def load_balancers(self, load_balancer_cfg):
"""Build ELB load balancers."""
existing_elb_names = self.evpc.elb.get_related_elb_names()
for lb_name, lb_cfg in load_balancer_cfg.items():
lb_fullname = "{}-{}".format(self.evpc.name, lb_name)
if lb_fullname in existing_elb_names:
msg = "skipping load_balancer: {} (it already exists)"
self.log.emit(msg.format(lb_fullname), "debug")
continue
self.log.emit("creating {} load_balancer ...".format(lb_fullname))
# make list of security group ids.
security_groups = map_filter_false(
self.evpc.get_security_group, lb_cfg.get("security_groups", [])
)
sg_ids = get_ids(security_groups)
# make list of subnet ids.
subnets = map_filter_false(self.evpc.get_subnet, lb_cfg.get("subnets", []))
sn_ids = get_ids(subnets)
scheme = "internet-facing"
if lb_cfg.get("internal", False):
scheme = "internal"
listeners = lb_cfg.get("listeners", [])
self.evpc.elb.create_load_balancer(
LoadBalancerName=lb_fullname,
Subnets=sn_ids,
SecurityGroups=sg_ids,
Scheme=scheme,
Tags=[
{"Key": "vpc_name", "Value": self.evpc.vpc_name},
{"Key": "role", "Value": lb_cfg["instance_role"]},
],
Listeners=self.evpc.elb.format_listeners(listeners),
)
self.log.emit("created {} load_balancer ...".format(lb_fullname))
self.log.emit(
"Configure Health Check for {} load_balancer ...".format(lb_fullname)
)
hc_cfg = lb_cfg.get("healthcheck", {})
hc_default_target = "TCP:{}".format(listeners[0][1])
self.evpc.elb.configure_health_check(
LoadBalancerName=lb_fullname,
HealthCheck={
"Target": hc_cfg.get("target", hc_default_target),
"Interval": hc_cfg.get("interval", 15),
"Timeout": hc_cfg.get("timeout", 5),
"UnhealthyThreshold": hc_cfg.get("unhealthy_threshold", 4),
"HealthyThreshold": hc_cfg.get("healthy_threshold", 4),
},
)
self.log.emit(
"Configured Health Check for {} load_balancer ...".format(lb_fullname)
)
asg_name = "{}-{}".format(self.evpc.name, lb_cfg["instance_role"])
if asg_name in self.evpc.autoscaling.get_related_autoscaling_group_names():
self.log.emit(
"attaching {} load balancer to {} autoscaling group ...".format(
lb_fullname, asg_name
)
)
self.evpc.autoscaling.attach_load_balancers(
AutoScalingGroupName=asg_name, LoadBalancerNames=[lb_fullname]
)
else:
self.log.emit(
"registering {} role to {} load balancer ...".format(
lb_cfg["instance_role"], lb_fullname
)
)
self.evpc.elb.register_role_with_load_balancer(
lb_fullname, lb_cfg["instance_role"]
)
def autoscaling_instance_role(self, role_name, role_data, desired_count):
if role_data.get("autoscaling", False) != True:
# exit early if this instance_role is not autoscaling.
return None
long_role_name = "{}-{}".format(self.evpc.name, role_name)
if (
long_role_name
in self.evpc.autoscaling.get_related_autoscaling_group_names()
):
msg = "skipping autoscaling group: {} (it already exists)"
self.log.emit(msg.format(long_role_name), "debug")
return None
ami = self.amis[role_data["ami"]][self.evpc.region_name]
key_pair = self.evpc.key_pair.get_key_pair(role_data.get("key_pair", "default"))
security_groups = map_filter_false(
self.evpc.get_security_group, role_data.get("security_groups", [])
)
subnets = map_filter_false(self.evpc.get_subnet, role_data.get("subnets", []))
block_device_map = get_block_device_map_from_role_config(role_data)
kwargs = {
"LaunchConfigurationName": long_role_name,
"ImageId": ami,
"InstanceType": role_data.get("instance_type"),
"KeyName": key_pair.name,
"SecurityGroups": get_ids(security_groups),
"BlockDeviceMappings": block_device_map,
"UserData": role_data.get("userdata", ""),
}
profile_name = role_data.get("instance_profile_name", None)
if profile_name:
# only add profile if set. Empty string causes error.
kwargs["IamInstanceProfile"] = profile_name
self.wait_for_instance_profile(profile_name)
self.log.emit(
"creating launch configuration for role: {}".format(long_role_name)
)
self.evpc.autoscaling.create_launch_configuration(**kwargs)
self.log.emit("creating autoscaling group for role: {}".format(long_role_name))
self.evpc.autoscaling.create_auto_scaling_group(
AutoScalingGroupName=long_role_name,
LaunchConfigurationName=long_role_name,
MinSize=desired_count,
MaxSize=desired_count,
DesiredCapacity=desired_count,
VPCZoneIdentifier=",".join(get_ids(subnets)),
Tags=[
{
"Key": "role",
"Value": role_name,
"Key": "Name",
"Value": long_role_name,
"PropagateAtLaunch": True,
}
],
)
def instance_role(self, role_name, role_data, desired_count):
if role_data.get("autoscaling", False) == True:
# exit early if this instance_role is autoscaling.
return None
self.log.emit("creating role: {}".format(role_name))
ami = self.amis[role_data["ami"]][self.evpc.region_name]
key_pair = self.evpc.key_pair.get_key_pair(role_data.get("key_pair", "default"))
security_groups = map_filter_false(
self.evpc.get_security_group, role_data.get("security_groups", [])
)
subnets = map_filter_false(self.evpc.get_subnet, role_data.get("subnets", []))
if len(subnets) == 0:
self.log.emit("no subnets found for role: {}".format(role_name), "warning")
# exit early.
return None
# sort by subnets by amount of instances, smallest first.
subnets = sorted(subnets, key=lambda sn: collection_len(sn.instances))
# determine the count of this role's existing instances.
# Note: we look for role in all subnets, not just the listed subnets.
existing_count = len(self.evpc.get_role(role_name))
if existing_count >= desired_count:
# for now we exit early, maybe terminate extras...
msg = "skipping role: {} (existing_count {} is greater than or equal to {})"
self.log.emit(msg.format(role_name, existing_count, desired_count), "debug")
return None
# determine count of additional instances needed to reach desired_count.
needed_count = desired_count - existing_count
needed_per_subnet = desired_count / len(subnets)
needed_remainder = desired_count % len(subnets)
block_device_map = get_block_device_map_from_role_config(role_data)
role_instances = []
kwargs = {
"ImageId": ami,
"InstanceType": role_data.get("instance_type"),
"KeyName": key_pair.name,
"SecurityGroupIds": get_ids(security_groups),
"BlockDeviceMappings": block_device_map,
"UserData": role_data.get("userdata", ""),
"IamInstanceProfile": {},
}
profile_name = role_data.get("instance_profile_name", None)
if profile_name:
kwargs["IamInstanceProfile"] = {"Name": profile_name}
private_ip_address = role_data.get("private_ip_address", None)
if private_ip_address:
kwargs["PrivateIpAddress"] = private_ip_address
for subnet in subnets:
# ensure Run_Instance_Idempotency.html#client-tokens
kwargs["ClientToken"] = str(uuid4())
# figure out how many instances this subnet needs to create ...
existing_in_subnet = len(
self.evpc.get_role(role_name, subnet.instances.all())
)
count = needed_per_subnet - existing_in_subnet
if needed_remainder != 0:
needed_remainder -= 1
count += 1
if count == 0:
# skip this subnet, it doesn't need to launch any instances.
continue
subnet_name = make_tag_dict(subnet)["Name"]
msg = "{} instances of role {} launching into {} subnet"
self.log.emit(msg.format(count, role_name, subnet_name))
# create a batch of instances in subnet!
kwargs["MinCount"] = kwargs["MaxCount"] = count
instances = self._create_instances(subnet, **kwargs)
# accumulate all new instances into a single list.
role_instances += instances
# add role tag to each instance.
for instance in role_instances:
update_tags(instance, role=role_name)
