def get_subnets_for_elb(self, scheme=None, env=None): vpc = Vpc() v = vpc.get_vpc_from_env(env=env) azs = Misc.get_azs_from_yaml(region=self.region) if scheme == "internal": avail = "private" else: avail = "public" res = [] sub = vpc.get_all_subnets( filters=[{'Name': 'tag:Availability', 'Values': [avail]}, {'Name': 'availabilityZone', 'Values': azs}, {'Name': 'vpc-id', 'Values': [v.get('VpcId')]}]) for s in sub: logger.debug("Adding sub: %s" % sub, ) res.append(s.get('SubnetId')) logger.debug("Subnets for elb are: %s" % res, ) return res
def deploy_snappy(self, env, num, dryrun, accountid, newrelic, channelname, devicestring, branch): from wrapper.ec2 import Ec2 from wrapper.vpc import Vpc ec2 = Ec2(session=self.session) vpc = Vpc(session=self.session) vpc_obj = vpc.get_vpc_from_env(env=env) num = int(num) snappyindex = self.get_snappy_index(num=num, vpcid=vpc_obj.get('VpcId')) lambda_function_args = { 'env': "infra", 'puppet_role': 'benchmarkslave', 'requester': "benchmark", 'xively_service': "benchmark_slave", 'customer': "", 'shutdown': "stop", 'dry_run': dryrun, 'base_ami': "benchmarkslave", 'instance_type': 'c3.xlarge', 'snappyindex': snappyindex, 'accountid': accountid, 'channelname': channelname, 'newrelic': newrelic, 'iam': 'infra-benchmarkslave', 'ebsoptimized': False, 'monitoring': False, 'devicestring': devicestring, 'branch': branch } lambda_function_args['userdata'] = Misc.get_userdata_for_os( ostype="snappy") baseami_object = self.get_ami_from_tag( puppet_role=lambda_function_args['base_ami']) lambda_function_args['baseamiid'] = baseami_object.get('ImageId') availability = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Availability') lambda_function_args['ostype'] = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Os') lambda_function_args['keypair'] = Misc.get_value_from_array_hash( dictlist=vpc_obj.get('Tags'), key='Keypair') lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name( env=env, puppet_role="benchmarkslave", num=num) lambda_function_args[ 'securitygroup'] = ec2.get_security_group_ids_for_stack( vpcid=vpc_obj.get('VpcId'), puppet_role="benchmarkslave", ostype=lambda_function_args['ostype'], xively_service="benchmark_slave") subnets = vpc.get_all_subnets(filters=[{ 'Name': 'tag:Network', 'Values': [availability] }, { 'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')] }]) lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym( puppet_role="benchmarkslave", subnets=subnets, num=num, fillup=False, xively_service="benchmark_slave") ## Get broker IP address broker = ec2.get_ec2_instances( filters=[{ 'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')] }, { 'Name': 'tag:Xively_service', 'Values': ['benchmark_master'] }, { 'Name': 'tag:Puppet_role', 'Values': ['linuxbase'] }]) lambda_function_args['broker'] = broker[0].get( 'PrivateIpAddress') + ":8883" instances = Misc.parallel_map_reduce( lambda x: self.create_instance_lamdba(args=lambda_function_args), lambda x, y: x + [y], xrange(0, num), []) return instances
def launch_auto_scaling_group(self, env=None, stack=None, min_size=None, max_size=None, xively_service=None, requester=None, load_balancer_name=None, health_check=None, health_check_grace_period=None, availability=None, customer=None): a = AutoScaling() v = Vpc() c = CloudWatch() logger.info("Starting creation of auto-scaling group") auto_scaling_group_name = a.generate_auto_scaling_group_name( env=env, stack=stack, xively_service=xively_service) launch_config_name = a.generate_launch_config_name( env=env, stack=stack, xively_service=xively_service) lc_exists = a.check_launch_config_exists(env=env, xively_service=xively_service, stack=stack) if lc_exists is False: logger.info("Starting to Create Launch Configuration: %s" % launch_config_name) a.create_launch_config(launch_config_name=launch_config_name, env=env, xively_service=xively_service, stack=stack) else: logger.info("Launch Configuration %s Already Exists" % launch_config_name) vpc = v.get_vpc_from_env(env=env) subnet_filter = v.get_all_subnets( filters=[{ "Name": "tag:Availability", "Values": [ availability, ] }, { "Name": "vpc-id", "Values": [ vpc.get('VpcId'), ] }]) vpc_zones = "" for s in subnet_filter: vpc_zones += str(s['SubnetId']) vpc_zones += str(",") tags = [{ "ResourceId": auto_scaling_group_name, "ResourceType": "auto-scaling-group", "PropagateAtLaunch": False, "Key": "Name", "Value": auto_scaling_group_name }, { "ResourceId": auto_scaling_group_name, "ResourceType": "auto-scaling-group", "PropagateAtLaunch": True, "Key": "Requester", "Value": requester }, { "ResourceId": auto_scaling_group_name, "ResourceType": "auto-scaling-group", "PropagateAtLaunch": True, "Key": "Puppet_role", "Value": stack }, { "ResourceId": auto_scaling_group_name, "ResourceType": "auto-scaling-group", "PropagateAtLaunch": True, "Key": "Xively_service", "Value": xively_service }, { "ResourceId": auto_scaling_group_name, "ResourceType": "auto-scaling-group", "PropagateAtLaunch": True, "Key": "Environment", "Value": env }, { "ResourceId": auto_scaling_group_name, "ResourceType": "auto-scaling-group", "PropagateAtLaunch": True, "Key": "Customer", "Value": customer }] asg = [] asg.append(auto_scaling_group_name) asg_exists = a.check_auto_scaling_group_exists( auto_scaling_group_name=asg) if asg_exists is False: logger.info("Starting to Create Auto Scaling Group: %s" % launch_config_name) a.run_auto_scaling_group( auto_scaling_group_name=auto_scaling_group_name, min_size=min_size, max_size=max_size, launch_config_name=launch_config_name, load_balancer_name=load_balancer_name, health_check=health_check, health_check_grace_period=health_check_grace_period, vpc_zones=vpc_zones, tags=tags) # resp = a.get_status_auto_scaling_group(auto_scaling_group_name=auto_scaling_group_name) # logger.info(resp) else: logger.info("Auto Scaling Group %s Already Exists" % launch_config_name) a.create_scaling_policy(env=env, stack=stack, xively_service=xively_service) c.create_alarm_for_auto_scaling_group(env=env, stack=stack, xively_service=xively_service)
def create_ec2_instance(self, puppet_role, env, requester, customer, xively_service, base_ami, iam, instance_type, dry_run, shutdown, monitoring, fillup, num, keypair, availability=None): """ This function creates an ec2 instance :param puppet_role: the Puppet_role that should be used :param env: the environment where we should provision to :param requester: the user/team requesting the machine :param customer: For future use only :param xively_service: the Xively_service that should be used :param base_ami: the base_ami that should be used. Can default to puppet_role :param iam: The iam role that should be attached, defaults to ec2-base :param instance_type: the type of instance requested :param dry_run: No changes should be done :param shutdown: The shutdown behavior to use :param monitoring: Should monitoring be enabled :param fillup: Should fillup algorithym be used or round robin :param num: the number of instances :return: a list of instance objects """ from wrapper.ec2 import Ec2 from wrapper.vpc import Vpc from misc import Misc from core.stackdata import stackdata stackdata_object = stackdata(session=self.session) ec2 = Ec2(session=self.session) vpc = Vpc(session=self.session) lambda_function_args = { 'env': env, 'puppet_role': puppet_role, 'requester': requester, 'xively_service': xively_service, 'customer': customer, 'shutdown': shutdown, 'dry_run': dry_run } stack_data = stackdata_object.get_stack_data( puppet_role=puppet_role, xively_service=xively_service) vpc_obj = vpc.get_vpc_from_env(env=env) ## Find the baseami object that needs to be used if base_ami: base_ami = base_ami elif 'ami' in stack_data: base_ami = stack_data['ami'] else: logger.info("Falling back to puppet_role as AMI name") base_ami = puppet_role logger.info("The baseami that is going to be used: %s" % (base_ami, )) baseami_object = self.get_ami_from_tag(puppet_role=base_ami) ## Get values for lambda function lambda_function_args['baseamiid'] = baseami_object.get('ImageId') if (availability == None): availability = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Availability') lambda_function_args['ostype'] = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Os') if keypair is not None: lambda_function_args['keypair'] = keypair else: lambda_function_args['keypair'] = Misc.get_value_from_array_hash( dictlist=vpc_obj.get('Tags'), key='Keypair') ## Find the instance_type that needs to be used if instance_type: inst_type_final = instance_type elif 'instance_type' in stack_data and env in stack_data[ 'instance_type']: inst_type_final = stack_data['instance_type'][env] else: inst_type_final = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Instancetype') logger.info("Instance type that will be used: %s" % (inst_type_final, )) lambda_function_args['instance_type'] = inst_type_final ## Find the instance profile that needs to be used if iam: iam_name = iam elif 'iam' in stack_data: iam_name = "%s-%s" % (env, stack_data['iam']['name_postfix']) else: iam_name = "ec2-base" logger.info("Base iam instance profile name: %s" % (iam_name, )) lambda_function_args['iam'] = iam_name ## Find value for ebsoptimized if 'ebsoptimized' in stack_data and env in stack_data['ebsoptimized']: lambda_function_args['ebsoptimized'] = Misc.str2bool( stack_data['ebsoptimized'][env]) else: lambda_function_args['ebsoptimized'] = False ## Find value for monitoring enablement if monitoring: lambda_function_args['monitoring'] = monitoring elif 'monitoring' in stack_data and env in stack_data['monitoring']: lambda_function_args['monitoring'] = Misc.str2bool( stack_data['monitoring'][env]) else: lambda_function_args['monitoring'] = False ## Generate instance names for all required instances lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name( env=env, puppet_role=puppet_role, num=num) ## Gather all security groups needed for creating an instance stack lambda_function_args[ 'securitygroup'] = ec2.get_security_group_ids_for_stack( vpcid=vpc_obj.get('VpcId'), puppet_role=puppet_role, ostype=lambda_function_args['ostype'], xively_service=xively_service) # We need to retrieve the subnets from Vpc object, and pass it to Ec2 object subnets = vpc.get_all_subnets(filters=[{ 'Name': 'tag:Network', 'Values': [availability] }, { 'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')] }]) lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym( puppet_role=puppet_role, subnets=subnets, num=num, fillup=fillup, xively_service=xively_service) instances = Misc.parallel_map_reduce( lambda x: self.create_instance_lamdba(args=lambda_function_args), lambda x, y: x + [y], xrange(0, num), []) return instances