def gather_information_for_cloudofrmation_parameters(stack_data, vpc, ami): parameters = [] env = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'), key="Environment") if 'cloudformation_parameters' in stack_data: for parameter in stack_data['cloudformation_parameters']: if parameter["ParameterKey"] == "Environment": parameters.append({"ParameterKey": "Environment", "ParameterValue": env, "UsePreviousValue": False}) elif parameter["ParameterKey"] == "InstanceType": instance = None if 'instance_type' in stack_data and env in stack_data['instance_type']: instance = stack_data["instance_type"][env] else: instance = Misc.get_value_from_array_hash(dictlist=ami.get('Tags'), key="Instancetype") parameters.append( {"ParameterKey": "InstanceType", "ParameterValue": instance, "UsePreviousValue": False}) elif parameter["ParameterKey"] == "Puppetrole": parameters.append({"ParameterKey": "Puppetrole", "ParameterValue": stack_data['puppet_role'], "UsePreviousValue": False}) elif parameter["ParameterKey"] == "XivelyService": parameters.append({"ParameterKey": "XivelyService", "ParameterValue": stack_data['xively_service'], "UsePreviousValue": False}) elif parameter["ParameterKey"] == "Ami": parameters.append( {"ParameterKey": "Ami", "ParameterValue": stack_data['ami'], "UsePreviousValue": False}) elif parameter["ParameterKey"] == "KeyName": key = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'), key="Keypair") parameters.append({"ParameterKey": "KeyName", "ParameterValue": key, "UsePreviousValue": False}) else: parameter["UsePreviousValue"] = False parameters.append(parameter) else: logger.warning(msg="No cloudformation parameter object in json") logger.debug(msg="Cloudformation parameters is: %s" % (parameters,)) return parameters
def info_all(self): elbs = self.get_all_elbs() result = [] yaml = Misc.get_aws_yaml(yaml_file="elb") V = Vpc() for lb in elbs: tmp = lb['LoadBalancerName'].split('-') if len(tmp) >= 3: elb_env = tmp.pop(0) short_env = tmp.pop(0) elb_stack = "-".join(tmp) elb_facing = lb['Scheme'] if elb_stack in yaml and elb_facing in yaml[elb_stack]: yaml_info = yaml[elb_stack][elb_facing] v = V.get_vpc_from_env(env=elb_env) domain = Misc.get_value_from_array_hash(dictlist=v.get('Tags'), key="Domain") if elb_facing == "internet-facing": elb_dns_name = yaml_info['dns'] + "." + Misc.change_domain_to_local(domain=domain) elif elb_facing == "internal": elb_dns_name = yaml_info['dns'] + "." + Misc.change_domain_to_local(domain=domain) else: elb_dns_name = None info = {} if elb_dns_name is not None: info['DNS cname'] = elb_dns_name else: info['DNS cname'] = "This elb is not in automatisation framework. Will be decomissioned" info['Xively_service'] = Misc.get_value_from_array_hash(dictlist=lb.get('Tags'), key="Xively_service") info['Puppet_role'] = Misc.get_value_from_array_hash(dictlist=lb.get('Tags'), key="Puppet_role") info['Env'] = Misc.get_value_from_array_hash(dictlist=lb.get('Tags'), key="Environment") info['Real endpoint'] = lb['DNSName'] info['Vpcid'] = lb['VPCId'] info['Name'] = lb['LoadBalancerName'] info['CreateTime'] = lb['CreatedTime'].strftime("%Y-%m-%d %H:%M") info['Facing'] = elb_facing info['Availability Zones'] = lb['AvailabilityZones'] info['Securitygroups'] = lb['SecurityGroups'] instance = [] for i in lb['Instances']: instance.append(i['InstanceId']) info['InstanceIds'] = instance listen = [] for listener in lb['ListenerDescriptions']: listener = listener['Listener'] listen.append( "%s-%s-%s" % (listener['LoadBalancerPort'], listener['InstancePort'], listener['Protocol'])) info['From-To-Protocol'] = listen result.append(info) return result
def route53_info(self, env=None): envs = super(awsrequests, self).get_needed_envs(env=env) v = Vpc() r = Route53() res = [] for environment in envs: logger.debug("Working in env: %s" % environment) vpc = v.get_vpc_from_env(env=environment) domain = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'), key="Domain") zoneid = r.get_zoneid_from_domain(domain=domain) records = r.list_zone_records(zoneid=zoneid) for record in records: rec = record.pop('ResourceRecords', []) values = [] for rr in rec: values.append(rr['Value']) record['Values'] = values if 'AliasTarget' in record: aliastarget = record.pop('AliasTarget') record['TTL'] = 'alias' record['Values'] = [aliastarget['DNSName']] record['Env'] = environment res.append(record) logger.debug("Processed record is: %s" % record, ) return res
def compare_elb_tags(elb_tags=None, tags=None): for tag in tags: elb_tag_value = Misc.get_value_from_array_hash(dictlist=elb_tags, key=tag) if elb_tag_value == tags[tag]: continue else: return False return True
def test_get_value_from_array_hash_invalid_key(self): array_hash = [{ 'Key': 'test', 'Value': 'test_value' }, { 'Key': 'test2', 'Value': 'test2_value' }] result = Misc.get_value_from_array_hash(dictlist=array_hash, key="test3") self.assertEqual(None, result)
def create_launch_config(self, launch_config_name=None, env=None, xively_service=None, stack=None): e = Ec2() v = Vpc() vpc = v.get_vpc_from_env(env=env) keyname = Misc.get_value_from_array_hash(dictlist=vpc['Tags'], key='Keypair') baseami = e.query_base_image(stack=stack) ostype = Misc.get_value_from_array_hash(dictlist=baseami['Tags'], key='Os') instance_type = Misc.get_value_from_array_hash( dictlist=baseami['Tags'], key='Instancetype') image = baseami.get('ImageId') sgs = e.get_security_group_ids_for_launch( vpcid=vpc.get('VpcId'), stack=stack, ostype=ostype, xively_service=xively_service) iam = "ec2" y = Misc.get_app_ports_yaml('app_ports') port = y[xively_service] userdata = Misc.get_autoscale_userdata_for_os(ostype=ostype).format( env=env, stack=stack, xively_service=xively_service, port=port) monitoring = {} monitoring['Enabled'] = True self.autoscale.create_launch_configuration( LaunchConfigurationName=launch_config_name, ImageId=image, KeyName=keyname, SecurityGroups=sgs, UserData=userdata, InstanceType=instance_type, InstanceMonitoring=monitoring, IamInstanceProfile=iam)
def get_server_cert_for_env(self, env=None): certs = self.get_server_certs() v = Vpc() vpc = v.get_vpc_from_env(env=env) domain = Misc.get_value_from_array_hash(dictlist=vpc['Tags'], key='Domain') cert_name = "star." + domain logger.debug("Searching for cert domain %s" % cert_name, ) for c in certs: logger.debug("Investigateing Cert %s" % c, ) if c['ServerCertificateName'] == cert_name: logger.debug("We found the server certificate we are looking for") return c logger.warning("Could not find the certificate for %s" % env, ) return None
def get_ami_stacks(self, account_id): ''' This function returns all active ami Puppet_roles :param account_id: The account id that is being used. IAM wrapper returns this number :type account_id: int :return: Array of strings of valid puppet_roles :rtype: array ''' images = self.get_images(account_id=account_id, filters=[{'Name': "tag-key", 'Values': ['Puppet_role']}]) stacks = {} for i in images: v = Misc.get_value_from_array_hash(dictlist=i['Tags'], key='Puppet_role') if v is not "" and v is not None: stacks[v] = 1 stacks = stacks.keys() logger.debug("Active stacks: " + str(stacks)) return stacks
def rds_instance_filters(rds=None, filters=None): for f in filters: logger.debug("Filter investigation %s" % f, ) if f['Name'] == "VpcId": if f['Values'][0] == rds.get('DBSubnetGroup').get('VpcId'): logger.info("This is the VPC we need for rds %s" % rds.get('DBSubnetGroup').get('VpcId'), ) else: logger.debug("RDS is in wrong VPC") return False if f['Name'] == "tag:Name": if 'Tags' in rds: logger.debug("RDS instance has tags") tag_name = Misc.get_value_from_array_hash(dictlist=rds['Tags'], key='Name') if f['Values'][0] == tag_name: logger.info("Tag name is same") continue return False return True
def create_elb(self, name=None, listeners=None, scheme=None, tags=None, env=None, sg_name=None): subnets = self.get_subnets_for_elb(scheme=scheme, env=env) yaml_tags = Misc.get_yaml_tags_for_sub(sub="elb") lb_name = self.generate_elb_name(stack=name, facing=scheme, env=env) for y in yaml_tags: logger.debug("Checking if tag exists %s" % y, ) if y == "Environment": tags.append({'Key': y, 'Value': env}) continue if y == "Name": tags.append({'Key': y, 'Value': lb_name}) continue t = Misc.get_value_from_array_hash(dictlist=tags, key=y) if t is None: tags.append({'Key': y, 'Value': ""}) sgs = self.get_sgs_for_elb(env=env, name=sg_name) self.elb.create_load_balancer(LoadBalancerName=lb_name, Scheme=scheme, Tags=tags, SecurityGroups=sgs, Subnets=subnets, Listeners=listeners) return lb_name
def get_all_subnets(self, filters=None, subnetids=None): """ This function returns all subnets, or filters them as requested :param filters: A dict list with the boto3 filters :param subnetids: A list of subnetids that should only be returned :return: A list of subnets that were requested """ if subnetids: response = self.vpc_client.describe_subnets(SubnetIds=subnetids) elif filters: response = self.vpc_client.describe_subnets(Filters=filters) else: response = self.vpc_client.describe_subnets() result = [] for s in response['Subnets']: allowed = Misc.get_value_from_array_hash(dictlist=s.get('Tags'), key="Allowed") if Misc.str2bool(allowed): result.append(s) logger.debug("Allowed az subnets are: %s" % (result,)) return result
def get_active_envs(self, env=None): """ This function returns an array with the active environemnts in the account :param env: a comma seperated list of environments that should be validated :type env: basestring :return: An array with active environments :rtype: list """ vpcs = self.get_all_vpcs() envs = [] for vpc in vpcs: cur = Misc.get_value_from_array_hash(dictlist=vpc['Tags'], key='Environment') if cur != "": envs.append(cur) else: logger.warning("Vpc has no Environment tag: %s" % (vpc.id)) if env: envs = [env] logger.debug("Current envs: " + str(envs)) return envs
def envs_with_domains(self): v = Vpc() envs = v.get_active_envs() res = {} for env in envs: res[env] = [] vpc = v.get_vpc_from_env(env=env) domain = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'), key='Domain') res[env].append(domain) logger.debug('Working on env %s and domain %s' % (env, domain)) split_domain = Misc.string_to_array(string=domain) last_dn = split_domain.pop() # local entry if env == "prod": local_tmp = ["prod"] + split_domain + ['local'] else: local_tmp = split_domain + ['local'] res[env].append( Misc.join_list_to_string(list=local_tmp, join_with='.')) return res
def get_all_subnets(self, filters=None, subnetids=None): """ This function returns all subnets, or filters them as requested :param filters: A dict list with the boto3 filters :param subnetids: A list of subnetids that should only be returned :return: A list of subnets that were requested """ if subnetids: response = self.vpc_client.describe_subnets(SubnetIds=subnetids) elif filters: response = self.vpc_client.describe_subnets(Filters=filters) else: response = self.vpc_client.describe_subnets() result = [] for s in response['Subnets']: allowed = Misc.get_value_from_array_hash(dictlist=s.get('Tags'), key="Allowed") if Misc.str2bool(allowed): result.append(s) logger.debug("Allowed az subnets are: %s" % (result, )) return result
def get_ami_stacks(self, account_id): ''' This function returns all active ami Puppet_roles :param account_id: The account id that is being used. IAM wrapper returns this number :type account_id: int :return: Array of strings of valid puppet_roles :rtype: array ''' images = self.get_images(account_id=account_id, filters=[{ 'Name': "tag-key", 'Values': ['Puppet_role'] }]) stacks = {} for i in images: v = Misc.get_value_from_array_hash(dictlist=i['Tags'], key='Puppet_role') if v is not "" and v is not None: stacks[v] = 1 stacks = stacks.keys() logger.debug("Active stacks: " + str(stacks)) return stacks
def get_all_image_instances(self, imageid): if imageid: instances = self.ami_client.describe_instances(Filters=[{'Name': 'image-id', 'Values': [imageid]}]) else: instances = self.ami_client.describe_instances() super(Ami, self).query_information(query=instances) temp = {} for reservation in instances['Reservations']: for instance in reservation['Instances']: if 'Tags' in instance: value = Misc.get_value_from_array_hash(dictlist=instance.get('Tags'), key='Name') else: value = instance.get('InstanceId') if instance['ImageId'] not in temp: temp[instance['ImageId']] = [value] else: temp[instance['ImageId']].append(value) ret = [] for imageid in temp: img = {'ImageId': imageid} img['InstanceCount'] = len(temp[imageid]) img['Instances'] = temp[imageid] ret.append(img) return ret
def gather_information_for_cloudofrmation_parameters(stack_data, vpc, ami): parameters = [] env = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'), key="Environment") if 'cloudformation_parameters' in stack_data: for parameter in stack_data['cloudformation_parameters']: if parameter["ParameterKey"] == "Environment": parameters.append({ "ParameterKey": "Environment", "ParameterValue": env, "UsePreviousValue": False }) elif parameter["ParameterKey"] == "InstanceType": instance = None if 'instance_type' in stack_data and env in stack_data[ 'instance_type']: instance = stack_data["instance_type"][env] else: instance = Misc.get_value_from_array_hash( dictlist=ami.get('Tags'), key="Instancetype") parameters.append({ "ParameterKey": "InstanceType", "ParameterValue": instance, "UsePreviousValue": False }) elif parameter["ParameterKey"] == "Puppetrole": parameters.append({ "ParameterKey": "Puppetrole", "ParameterValue": stack_data['puppet_role'], "UsePreviousValue": False }) elif parameter["ParameterKey"] == "XivelyService": parameters.append({ "ParameterKey": "XivelyService", "ParameterValue": stack_data['xively_service'], "UsePreviousValue": False }) elif parameter["ParameterKey"] == "Ami": parameters.append({ "ParameterKey": "Ami", "ParameterValue": stack_data['ami'], "UsePreviousValue": False }) elif parameter["ParameterKey"] == "KeyName": key = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'), key="Keypair") parameters.append({ "ParameterKey": "KeyName", "ParameterValue": key, "UsePreviousValue": False }) else: parameter["UsePreviousValue"] = False parameters.append(parameter) else: logger.warning(msg="No cloudformation parameter object in json") logger.debug(msg="Cloudformation parameters is: %s" % (parameters, )) return parameters
def create_ec2_instance(self, puppet_role, env, requester, customer, xively_service, base_ami, iam, instance_type, dry_run, shutdown, monitoring, fillup, num, keypair, availability=None): """ This function creates an ec2 instance :param puppet_role: the Puppet_role that should be used :param env: the environment where we should provision to :param requester: the user/team requesting the machine :param customer: For future use only :param xively_service: the Xively_service that should be used :param base_ami: the base_ami that should be used. Can default to puppet_role :param iam: The iam role that should be attached, defaults to ec2-base :param instance_type: the type of instance requested :param dry_run: No changes should be done :param shutdown: The shutdown behavior to use :param monitoring: Should monitoring be enabled :param fillup: Should fillup algorithym be used or round robin :param num: the number of instances :return: a list of instance objects """ from wrapper.ec2 import Ec2 from wrapper.vpc import Vpc from misc import Misc from core.stackdata import stackdata stackdata_object = stackdata(session=self.session) ec2 = Ec2(session=self.session) vpc = Vpc(session=self.session) lambda_function_args = { 'env': env, 'puppet_role': puppet_role, 'requester': requester, 'xively_service': xively_service, 'customer': customer, 'shutdown': shutdown, 'dry_run': dry_run } stack_data = stackdata_object.get_stack_data( puppet_role=puppet_role, xively_service=xively_service) vpc_obj = vpc.get_vpc_from_env(env=env) ## Find the baseami object that needs to be used if base_ami: base_ami = base_ami elif 'ami' in stack_data: base_ami = stack_data['ami'] else: logger.info("Falling back to puppet_role as AMI name") base_ami = puppet_role logger.info("The baseami that is going to be used: %s" % (base_ami, )) baseami_object = self.get_ami_from_tag(puppet_role=base_ami) ## Get values for lambda function lambda_function_args['baseamiid'] = baseami_object.get('ImageId') if (availability == None): availability = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Availability') lambda_function_args['ostype'] = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Os') if keypair is not None: lambda_function_args['keypair'] = keypair else: lambda_function_args['keypair'] = Misc.get_value_from_array_hash( dictlist=vpc_obj.get('Tags'), key='Keypair') ## Find the instance_type that needs to be used if instance_type: inst_type_final = instance_type elif 'instance_type' in stack_data and env in stack_data[ 'instance_type']: inst_type_final = stack_data['instance_type'][env] else: inst_type_final = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Instancetype') logger.info("Instance type that will be used: %s" % (inst_type_final, )) lambda_function_args['instance_type'] = inst_type_final ## Find the instance profile that needs to be used if iam: iam_name = iam elif 'iam' in stack_data: iam_name = "%s-%s" % (env, stack_data['iam']['name_postfix']) else: iam_name = "ec2-base" logger.info("Base iam instance profile name: %s" % (iam_name, )) lambda_function_args['iam'] = iam_name ## Find value for ebsoptimized if 'ebsoptimized' in stack_data and env in stack_data['ebsoptimized']: lambda_function_args['ebsoptimized'] = Misc.str2bool( stack_data['ebsoptimized'][env]) else: lambda_function_args['ebsoptimized'] = False ## Find value for monitoring enablement if monitoring: lambda_function_args['monitoring'] = monitoring elif 'monitoring' in stack_data and env in stack_data['monitoring']: lambda_function_args['monitoring'] = Misc.str2bool( stack_data['monitoring'][env]) else: lambda_function_args['monitoring'] = False ## Generate instance names for all required instances lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name( env=env, puppet_role=puppet_role, num=num) ## Gather all security groups needed for creating an instance stack lambda_function_args[ 'securitygroup'] = ec2.get_security_group_ids_for_stack( vpcid=vpc_obj.get('VpcId'), puppet_role=puppet_role, ostype=lambda_function_args['ostype'], xively_service=xively_service) # We need to retrieve the subnets from Vpc object, and pass it to Ec2 object subnets = vpc.get_all_subnets(filters=[{ 'Name': 'tag:Network', 'Values': [availability] }, { 'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')] }]) lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym( puppet_role=puppet_role, subnets=subnets, num=num, fillup=fillup, xively_service=xively_service) instances = Misc.parallel_map_reduce( lambda x: self.create_instance_lamdba(args=lambda_function_args), lambda x, y: x + [y], xrange(0, num), []) return instances
def deploy_snappy(self, env, num, dryrun, accountid, newrelic, channelname, devicestring, branch): from wrapper.ec2 import Ec2 from wrapper.vpc import Vpc ec2 = Ec2(session=self.session) vpc = Vpc(session=self.session) vpc_obj = vpc.get_vpc_from_env(env=env) num = int(num) snappyindex = self.get_snappy_index(num=num, vpcid=vpc_obj.get('VpcId')) lambda_function_args = { 'env': "infra", 'puppet_role': 'benchmarkslave', 'requester': "benchmark", 'xively_service': "benchmark_slave", 'customer': "", 'shutdown': "stop", 'dry_run': dryrun, 'base_ami': "benchmarkslave", 'instance_type': 'c3.xlarge', 'snappyindex': snappyindex, 'accountid': accountid, 'channelname': channelname, 'newrelic': newrelic, 'iam': 'infra-benchmarkslave', 'ebsoptimized': False, 'monitoring': False, 'devicestring': devicestring, 'branch': branch } lambda_function_args['userdata'] = Misc.get_userdata_for_os( ostype="snappy") baseami_object = self.get_ami_from_tag( puppet_role=lambda_function_args['base_ami']) lambda_function_args['baseamiid'] = baseami_object.get('ImageId') availability = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Availability') lambda_function_args['ostype'] = Misc.get_value_from_array_hash( dictlist=baseami_object.get('Tags'), key='Os') lambda_function_args['keypair'] = Misc.get_value_from_array_hash( dictlist=vpc_obj.get('Tags'), key='Keypair') lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name( env=env, puppet_role="benchmarkslave", num=num) lambda_function_args[ 'securitygroup'] = ec2.get_security_group_ids_for_stack( vpcid=vpc_obj.get('VpcId'), puppet_role="benchmarkslave", ostype=lambda_function_args['ostype'], xively_service="benchmark_slave") subnets = vpc.get_all_subnets(filters=[{ 'Name': 'tag:Network', 'Values': [availability] }, { 'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')] }]) lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym( puppet_role="benchmarkslave", subnets=subnets, num=num, fillup=False, xively_service="benchmark_slave") ## Get broker IP address broker = ec2.get_ec2_instances( filters=[{ 'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')] }, { 'Name': 'tag:Xively_service', 'Values': ['benchmark_master'] }, { 'Name': 'tag:Puppet_role', 'Values': ['linuxbase'] }]) lambda_function_args['broker'] = broker[0].get( 'PrivateIpAddress') + ":8883" instances = Misc.parallel_map_reduce( lambda x: self.create_instance_lamdba(args=lambda_function_args), lambda x, y: x + [y], xrange(0, num), []) return instances
def test_get_value_from_array_hash_empty_array(self): array_hash = [] result = Misc.get_value_from_array_hash(dictlist=array_hash, key="test3") self.assertEqual(None, result)
def test_get_value_from_array_hash_invalid_key(self): array_hash = [{'Key': 'test', 'Value': 'test_value'}, {'Key': 'test2', 'Value': 'test2_value'}] result = Misc.get_value_from_array_hash(dictlist=array_hash, key="test3") self.assertEqual(None, result)