def autoscaling_instance_role(self, role_name, role_data, desired_count): if role_data.get('autoscaling', False) != True: # exit early if this instance_role is not autoscaling. return None long_role_name = '{}-{}'.format(self.evpc.name, role_name) ami = self.amis[role_data['ami']][self.evpc.region_name] key_pair = self.evpc.key_pair.get_key_pair( role_data.get('key_pair', 'default')) security_groups = map_filter_false( self.evpc.get_security_group, role_data.get('security_groups', [])) subnets = map_filter_false(self.evpc.get_subnet, role_data.get('subnets', [])) block_device_map = get_block_device_map_from_role_config(role_data) kwargs = { 'LaunchConfigurationName': long_role_name, 'ImageId': ami, 'InstanceType': role_data.get('instance_type'), 'KeyName': key_pair.name, 'SecurityGroups': get_ids(security_groups), 'BlockDeviceMappings': block_device_map, 'UserData': role_data.get('userdata', ''), } profile_name = role_data.get('instance_profile_name', None) if profile_name: # only add profile if set. Empty string causes error. kwargs['IamInstanceProfile'] = profile_name self.wait_for_instance_profile(profile_name) self.log.emit('creating launch configuration for role: {}'.format( long_role_name)) self.evpc.autoscaling.create_launch_configuration(**kwargs) self.log.emit( 'creating autoscaling group for role: {}'.format(long_role_name)) self.evpc.autoscaling.create_auto_scaling_group( AutoScalingGroupName=long_role_name, LaunchConfigurationName=long_role_name, MinSize=desired_count, MaxSize=desired_count, DesiredCapacity=desired_count, VPCZoneIdentifier=','.join(get_ids(subnets)), Tags=[ { 'Key': 'role', 'Value': role_name, 'PropagateAtLaunch': True, }, ])
def db_instances(self, db_instance_cfg): """Build RDS DB Instances.""" existing_rds_names = self.evpc.rds.get_related_db_ids() for rds_name, db_cfg in db_instance_cfg.items(): if rds_name in existing_rds_names: msg = "skipping RDS db_instance: {} (it already exists)" self.log.emit(msg.format(rds_name), "debug") continue self.log.emit("creating {} RDS db_instance ...".format(rds_name)) # make list of security group ids. security_groups = map_filter_false( self.evpc.get_security_group, db_cfg.get("security_groups", []) ) sg_ids = get_ids(security_groups) # make list of subnet ids. subnets = map_filter_false(self.evpc.get_subnet, db_cfg.get("subnets", [])) sn_ids = get_ids(subnets) self.evpc.rds.create_db_subnet_group( DBSubnetGroupName=rds_name, DBSubnetGroupDescription=db_cfg.get("description", ""), SubnetIds=sn_ids, ) self.evpc.rds.create_db_instance( DBInstanceIdentifier=rds_name, DBSubnetGroupName=rds_name, DBName=db_cfg.get("name", rds_name), VpcSecurityGroupIds=sg_ids, DBInstanceClass=db_cfg.get("class", "db.t2.medium"), AllocatedStorage=db_cfg.get("allocated_storage", 100), Engine=db_cfg.get("engine"), EngineVersion=db_cfg.get("engine_version", ""), Iops=db_cfg.get("iops", 0), MultiAZ=db_cfg.get("multi_az", False), MasterUsername=db_cfg.get("master_username"), MasterUserPassword=generate_password(16), BackupRetentionPeriod=db_cfg.get("backup_retention_period", 0), StorageType=db_cfg.get("storage_type", "standard"), # 'gp2' StorageEncrypted=db_cfg.get("storage_encryption", False), Tags=[{"Key": "vpc_name", "Value": self.evpc.vpc_name}], )
def db_instances(self, db_instance_cfg): """Build RDS DB Instances.""" for rds_name, db_cfg in db_instance_cfg.items(): self.log.emit('creating {} RDS db_instance ...'.format(rds_name)) # make list of security group ids. security_groups = map_filter_false( self.evpc.get_security_group, db_cfg.get('security_groups', [])) sg_ids = get_ids(security_groups) # make list of subnet ids. subnets = map_filter_false(self.evpc.get_subnet, db_cfg.get('subnets', [])) sn_ids = get_ids(subnets) self.evpc.rds.create_db_subnet_group( DBSubnetGroupName=rds_name, DBSubnetGroupDescription=db_cfg.get('description', ''), SubnetIds=sn_ids, ) self.evpc.rds.create_db_instance( DBInstanceIdentifier=rds_name, DBSubnetGroupName=rds_name, DBName=db_cfg.get('name', rds_name), VpcSecurityGroupIds=sg_ids, DBInstanceClass=db_cfg.get('class', 'db.t2.medium'), AllocatedStorage=db_cfg.get('allocated_storage', 100), Engine=db_cfg.get('engine'), EngineVersion=db_cfg.get('engine_version', ''), Iops=db_cfg.get('iops', 0), MultiAZ=db_cfg.get('multi_az', False), MasterUsername=db_cfg.get('master_username'), MasterUserPassword=generate_password(16), BackupRetentionPeriod=db_cfg.get('backup_retention_period', 0), Tags=[{ 'Key': 'vpc_name', 'Value': self.evpc.vpc_name }], )
def load_balancers(self, load_balancer_cfg): """Build ELB load balancers.""" for lb_name, lb_cfg in load_balancer_cfg.items(): lb_fullname = '{}-{}'.format(self.evpc.name, lb_name) self.log.emit('creating {} load_balancer ...'.format(lb_fullname)) # make list of security group ids. security_groups = map_filter_false( self.evpc.get_security_group, lb_cfg.get('security_groups', [])) sg_ids = get_ids(security_groups) # make list of subnet ids. subnets = map_filter_false(self.evpc.get_subnet, lb_cfg.get('subnets', [])) sn_ids = get_ids(subnets) scheme = 'internet-facing' if lb_cfg.get('internal', False): scheme = 'internal' listeners = lb_cfg.get('listeners', []) self.evpc.elb.create_load_balancer( LoadBalancerName=lb_fullname, Subnets=sn_ids, SecurityGroups=sg_ids, Scheme=scheme, Tags=[ { 'Key': 'vpc_name', 'Value': self.evpc.vpc_name }, { 'Key': 'role', 'Value': lb_cfg['instance_role'] }, ], Listeners=self.evpc.elb.format_listeners(listeners), ) self.log.emit('created {} load_balancer ...'.format(lb_fullname)) self.log.emit( 'Configure Health Check for {} load_balancer ...'.format( lb_fullname)) hc_cfg = lb_cfg.get('healthcheck', {}) hc_default_target = 'TCP:{}'.format(listeners[0][1]) self.evpc.elb.configure_health_check( LoadBalancerName=lb_fullname, HealthCheck={ 'Target': hc_cfg.get('target', hc_default_target), 'Interval': hc_cfg.get('interval', 15), 'Timeout': hc_cfg.get('timeout', 5), 'UnhealthyThreshold': hc_cfg.get('unhealthy_threshold', 4), 'HealthyThreshold': hc_cfg.get('healthy_threshold', 4) }) self.log.emit( 'Configured Health Check for {} load_balancer ...'.format( lb_fullname)) asg_name = '{}-{}'.format(self.evpc.name, lb_cfg['instance_role']) if asg_name in self.evpc.autoscaling.get_related_autoscaling_group_names( ): self.log.emit( 'attaching {} load balancer to {} autoscaling group ...'. format(lb_fullname, asg_name)) self.evpc.autoscaling.attach_load_balancers( AutoScalingGroupName=asg_name, LoadBalancerNames=[lb_fullname], ) else: self.log.emit( 'registering {} role to {} load balancer ...'.format( lb_cfg['instance_role'], lb_fullname)) self.evpc.elb.register_role_with_load_balancer( lb_fullname, lb_cfg['instance_role'])
def instance_role(self, role_name, role_data, desired_count): if role_data.get('autoscaling', False) == True: # exit early if this instance_role is autoscaling. return None self.log.emit('creating role: {}'.format(role_name)) ami = self.amis[role_data['ami']][self.evpc.region_name] key_pair = self.evpc.key_pair.get_key_pair( role_data.get('key_pair', 'default')) security_groups = map_filter_false( self.evpc.get_security_group, role_data.get('security_groups', [])) subnets = map_filter_false(self.evpc.get_subnet, role_data.get('subnets', [])) if len(subnets) == 0: self.log.emit('no subnets found for role: {}'.format(role_name), 'warning') # exit early. return None # sort by subnets by amount of instances, smallest first. subnets = sorted( subnets, key=lambda sn: collection_len(sn.instances), ) # determine the count of this role's existing instances. # Note: we look for role in all subnets, not just the listed subnets. existing_count = len(self.evpc.get_role(role_name)) if existing_count >= desired_count: # for now we exit early, maybe terminate extras... msg = 'skipping role: {} (existing_count {} is greater than or equal to {})' self.log.emit(msg.format(role_name, existing_count, desired_count), 'debug') return None # determine count of additional instances needed to reach desired_count. needed_count = desired_count - existing_count needed_per_subnet = desired_count / len(subnets) needed_remainder = desired_count % len(subnets) block_device_map = get_block_device_map_from_role_config(role_data) role_instances = [] kwargs = { 'ImageId': ami, 'InstanceType': role_data.get('instance_type'), 'KeyName': key_pair.name, 'SecurityGroupIds': get_ids(security_groups), 'BlockDeviceMappings': block_device_map, 'UserData': role_data.get('userdata', ''), 'IamInstanceProfile': {}, } profile_name = role_data.get('instance_profile_name', None) if profile_name: kwargs['IamInstanceProfile'] = {'Name': profile_name} for subnet in subnets: # ensure Run_Instance_Idempotency.html#client-tokens kwargs['ClientToken'] = str(uuid4()) # figure out how many instances this subnet needs to create ... existing_in_subnet = len( self.evpc.get_role(role_name, subnet.instances.all())) count = needed_per_subnet - existing_in_subnet if needed_remainder != 0: needed_remainder -= 1 count += 1 if count == 0: # skip this subnet, it doesn't need to launch any instances. continue subnet_name = make_tag_dict(subnet)['Name'] msg = '{} instances of role {} launching into {} subnet' self.log.emit(msg.format(count, role_name, subnet_name)) # create a batch of instances in subnet! kwargs['MinCount'] = kwargs['MaxCount'] = count instances = self._create_instances(subnet, **kwargs) # accumulate all new instances into a single list. role_instances += instances # add role tag to each instance. for instance in role_instances: update_tags(instance, role=role_name)
def load_balancers(self, load_balancer_cfg): """Build ELB load balancers.""" existing_elb_names = self.evpc.elb.get_related_elb_names() for lb_name, lb_cfg in load_balancer_cfg.items(): lb_fullname = "{}-{}".format(self.evpc.name, lb_name) if lb_fullname in existing_elb_names: msg = "skipping load_balancer: {} (it already exists)" self.log.emit(msg.format(lb_fullname), "debug") continue self.log.emit("creating {} load_balancer ...".format(lb_fullname)) # make list of security group ids. security_groups = map_filter_false( self.evpc.get_security_group, lb_cfg.get("security_groups", []) ) sg_ids = get_ids(security_groups) # make list of subnet ids. subnets = map_filter_false(self.evpc.get_subnet, lb_cfg.get("subnets", [])) sn_ids = get_ids(subnets) scheme = "internet-facing" if lb_cfg.get("internal", False): scheme = "internal" listeners = lb_cfg.get("listeners", []) self.evpc.elb.create_load_balancer( LoadBalancerName=lb_fullname, Subnets=sn_ids, SecurityGroups=sg_ids, Scheme=scheme, Tags=[ {"Key": "vpc_name", "Value": self.evpc.vpc_name}, {"Key": "role", "Value": lb_cfg["instance_role"]}, ], Listeners=self.evpc.elb.format_listeners(listeners), ) self.log.emit("created {} load_balancer ...".format(lb_fullname)) self.log.emit( "Configure Health Check for {} load_balancer ...".format(lb_fullname) ) hc_cfg = lb_cfg.get("healthcheck", {}) hc_default_target = "TCP:{}".format(listeners[0][1]) self.evpc.elb.configure_health_check( LoadBalancerName=lb_fullname, HealthCheck={ "Target": hc_cfg.get("target", hc_default_target), "Interval": hc_cfg.get("interval", 15), "Timeout": hc_cfg.get("timeout", 5), "UnhealthyThreshold": hc_cfg.get("unhealthy_threshold", 4), "HealthyThreshold": hc_cfg.get("healthy_threshold", 4), }, ) self.log.emit( "Configured Health Check for {} load_balancer ...".format(lb_fullname) ) asg_name = "{}-{}".format(self.evpc.name, lb_cfg["instance_role"]) if asg_name in self.evpc.autoscaling.get_related_autoscaling_group_names(): self.log.emit( "attaching {} load balancer to {} autoscaling group ...".format( lb_fullname, asg_name ) ) self.evpc.autoscaling.attach_load_balancers( AutoScalingGroupName=asg_name, LoadBalancerNames=[lb_fullname] ) else: self.log.emit( "registering {} role to {} load balancer ...".format( lb_cfg["instance_role"], lb_fullname ) ) self.evpc.elb.register_role_with_load_balancer( lb_fullname, lb_cfg["instance_role"] )
def autoscaling_instance_role(self, role_name, role_data, desired_count): if role_data.get("autoscaling", False) != True: # exit early if this instance_role is not autoscaling. return None long_role_name = "{}-{}".format(self.evpc.name, role_name) if ( long_role_name in self.evpc.autoscaling.get_related_autoscaling_group_names() ): msg = "skipping autoscaling group: {} (it already exists)" self.log.emit(msg.format(long_role_name), "debug") return None ami = self.amis[role_data["ami"]][self.evpc.region_name] key_pair = self.evpc.key_pair.get_key_pair(role_data.get("key_pair", "default")) security_groups = map_filter_false( self.evpc.get_security_group, role_data.get("security_groups", []) ) subnets = map_filter_false(self.evpc.get_subnet, role_data.get("subnets", [])) block_device_map = get_block_device_map_from_role_config(role_data) kwargs = { "LaunchConfigurationName": long_role_name, "ImageId": ami, "InstanceType": role_data.get("instance_type"), "KeyName": key_pair.name, "SecurityGroups": get_ids(security_groups), "BlockDeviceMappings": block_device_map, "UserData": role_data.get("userdata", ""), } profile_name = role_data.get("instance_profile_name", None) if profile_name: # only add profile if set. Empty string causes error. kwargs["IamInstanceProfile"] = profile_name self.wait_for_instance_profile(profile_name) self.log.emit( "creating launch configuration for role: {}".format(long_role_name) ) self.evpc.autoscaling.create_launch_configuration(**kwargs) self.log.emit("creating autoscaling group for role: {}".format(long_role_name)) self.evpc.autoscaling.create_auto_scaling_group( AutoScalingGroupName=long_role_name, LaunchConfigurationName=long_role_name, MinSize=desired_count, MaxSize=desired_count, DesiredCapacity=desired_count, VPCZoneIdentifier=",".join(get_ids(subnets)), Tags=[ { "Key": "role", "Value": role_name, "Key": "Name", "Value": long_role_name, "PropagateAtLaunch": True, } ], )
def instance_role(self, role_name, role_data, desired_count): if role_data.get("autoscaling", False) == True: # exit early if this instance_role is autoscaling. return None self.log.emit("creating role: {}".format(role_name)) ami = self.amis[role_data["ami"]][self.evpc.region_name] key_pair = self.evpc.key_pair.get_key_pair(role_data.get("key_pair", "default")) security_groups = map_filter_false( self.evpc.get_security_group, role_data.get("security_groups", []) ) subnets = map_filter_false(self.evpc.get_subnet, role_data.get("subnets", [])) if len(subnets) == 0: self.log.emit("no subnets found for role: {}".format(role_name), "warning") # exit early. return None # sort by subnets by amount of instances, smallest first. subnets = sorted(subnets, key=lambda sn: collection_len(sn.instances)) # determine the count of this role's existing instances. # Note: we look for role in all subnets, not just the listed subnets. existing_count = len(self.evpc.get_role(role_name)) if existing_count >= desired_count: # for now we exit early, maybe terminate extras... msg = "skipping role: {} (existing_count {} is greater than or equal to {})" self.log.emit(msg.format(role_name, existing_count, desired_count), "debug") return None # determine count of additional instances needed to reach desired_count. needed_count = desired_count - existing_count needed_per_subnet = desired_count / len(subnets) needed_remainder = desired_count % len(subnets) block_device_map = get_block_device_map_from_role_config(role_data) role_instances = [] kwargs = { "ImageId": ami, "InstanceType": role_data.get("instance_type"), "KeyName": key_pair.name, "SecurityGroupIds": get_ids(security_groups), "BlockDeviceMappings": block_device_map, "UserData": role_data.get("userdata", ""), "IamInstanceProfile": {}, } profile_name = role_data.get("instance_profile_name", None) if profile_name: kwargs["IamInstanceProfile"] = {"Name": profile_name} private_ip_address = role_data.get("private_ip_address", None) if private_ip_address: kwargs["PrivateIpAddress"] = private_ip_address for subnet in subnets: # ensure Run_Instance_Idempotency.html#client-tokens kwargs["ClientToken"] = str(uuid4()) # figure out how many instances this subnet needs to create ... existing_in_subnet = len( self.evpc.get_role(role_name, subnet.instances.all()) ) count = needed_per_subnet - existing_in_subnet if needed_remainder != 0: needed_remainder -= 1 count += 1 if count == 0: # skip this subnet, it doesn't need to launch any instances. continue subnet_name = make_tag_dict(subnet)["Name"] msg = "{} instances of role {} launching into {} subnet" self.log.emit(msg.format(count, role_name, subnet_name)) # create a batch of instances in subnet! kwargs["MinCount"] = kwargs["MaxCount"] = count instances = self._create_instances(subnet, **kwargs) # accumulate all new instances into a single list. role_instances += instances # add role tag to each instance. for instance in role_instances: update_tags(instance, role=role_name)