class Cloudability: #constructor def __init__(self): self.cloudability_dict = {} self.ah_obj = AtlasHelper() self.aws_helper_object = AwsHelper() self.module = "cloudability_module" self.auth_token = os.environ.get('CLOUDABILITY_AUTH_TOKEN') self.cl_base_url = self.ah_obj.get_atlas_config_data(self.module, "cloudability_base_url") self.cl_cost_url = self.ah_obj.get_atlas_config_data(self.module, "cloudability_cost_url") self.report_query = "" self.memcache_var = memcache.Client([self.ah_obj.get_atlas_config_data("global_config_data", 'memcache_server_location') ], debug=1) self.environment_subnets_details = self.aws_helper_object.get_environment_subnets_details() def construct_cost_query(self, query_parameters): try: self.report_query = self.cl_base_url+self.cl_cost_url+query_parameters+self.auth_token return self.report_query except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "construct_cost_query()", exp_object, exc_type, exc_obj, exc_tb) return def generate_report(self, query): try: report_json = {} response = requests.get(query) if response.status_code == 200: report_json = json.loads(response.text) #convert the json into a python dictionary return report_json except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_previous_period(self, start_date, end_date): try: start = datetime.datetime.strptime(start_date, '%Y-%m-%d') end = datetime.datetime.strptime(end_date, '%Y-%m-%d') period = ((end - start).days)+1 previous_start_date = (start-datetime.timedelta(days=period)).strftime('%Y-%m-%d') previous_end_date = (end-datetime.timedelta(days=period)).strftime('%Y-%m-%d') return (previous_start_date, previous_end_date) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_previous_period()", exp_object, exc_type, exc_obj, exc_tb) return #get ec2 costs def get_ec2_costs(self, start_date, end_date): ec2_costs={'region_zone': 0.0} try: query_parameters = "verbose=1&start_date="+start_date+"&end_date="+end_date+"&dimensions=linked_account_name&metrics=invoiced_cost&sort_by=invoiced_cost&order=desc&max_results=50&offset=0&auth_token=" ec2_costs_query = self.construct_cost_query(query_parameters) ec2_cost_dict = self.generate_report(ec2_costs_query) if ec2_cost_dict: ec2_costs['region_zone']= round(float(ec2_cost_dict['meta']['aggregates'][0]['value'].strip('$').replace(',','')),2) if ec2_costs['region_zone']: return ec2_costs else: raise Exception("Could not generate EC2 costs") except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "calculate_ec2_costs()", exp_object, exc_type, exc_obj, exc_tb) return ec2_costs #get aggregate cost spent for ec2 for the specified period def get_current_prev_ec2_costs(self, start_date, end_date): #query parameters should be moved to config file ec2_cost_dict = {'current_period': 0.0, 'previous_period': 0.0} try: previous_period = self.get_previous_period(start_date,end_date) previous_start_date, previous_end_date = previous_period[0], previous_period[1] ec2_cost_dict['current_period'] = self.get_ec2_costs(start_date, end_date) ec2_cost_dict['previous_period'] = self.get_ec2_costs(previous_start_date, previous_end_date) return ec2_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_ec2_costs()", exp_object, exc_type, exc_obj, exc_tb) return def create_envcost_dict(self): try: env_subnet_zip = self.environment_subnets_details per_environment_costs = {} for env_subnet_tuple in env_subnet_zip: per_environment_costs[env_subnet_tuple[0]] = 0 return per_environment_costs except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "create_envcost_dict()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_environment_costs(self, start_date, end_date): try: query_parameters = "verbose=1&start_date="+start_date+"&end_date="+end_date+"&dimensions=tag2&metrics=invoiced_cost&sort_by=invoiced_cost&order=desc&auth_token=" subnets_cost_query = self.construct_cost_query(query_parameters) subnet_cost_dict = self.generate_report(subnets_cost_query) cost_dict = self.create_envcost_dict() env_subnet_zip = self.environment_subnets_details subnet_details = subnet_cost_dict['results'] for subnet_index in subnet_details: for env_subnet_tuple in env_subnet_zip: if subnet_index['tag2'] in env_subnet_tuple[1]: #strip of the $ symbol and , convert the string to float with 2 precisions subnet_cost = float((subnet_index['invoiced_cost'].strip('$')).strip(',').replace(",","")) env_cost = cost_dict[env_subnet_tuple[0]] if subnet_index['tag2'] in cost_dict.keys(): cost_dict[env_subnet_tuple[0]]+= round((env_cost+subnet_cost),2) else: cost_dict[env_subnet_tuple[0]] = round((env_cost+subnet_cost),2) if cost_dict: return cost_dict else: raise Exception('Could not calculate environment costs') except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_environment_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_current_prev_environment_costs(self, start_date, end_date): #query parameters should be moved to config file try: env_cost_dict = {} previous_period = self.get_previous_period(start_date,end_date) previous_start_date, previous_end_date = previous_period[0], previous_period[1] env_cost_dict['current_period'] = self.get_environment_costs(start_date, end_date) env_cost_dict['previous_period'] = self.get_environment_costs(previous_start_date, previous_end_date) return env_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return #calculate costs subnet wise def get_subnet_costs(self, start_date, end_date): subnet_cost_dict = {} try: query_parameters = "verbose=1&start_date="+start_date+"&end_date="+end_date+"&dimensions=tag2&metrics=invoiced_cost&sort_by=invoiced_cost&order=desc&auth_token=" subnets_cost_query = self.construct_cost_query(query_parameters) subnet_cost_json = self.generate_report(subnets_cost_query) env_subnet_zip = self.environment_subnets_details subnet_details = subnet_cost_json['results'] for subnet_index in subnet_details: for env_subnet_tuple in env_subnet_zip: if subnet_index['tag2'] in env_subnet_tuple[1]: #strip of the $ symbol and , convert the string to float with 2 precisions subnet_cost = float((subnet_index['invoiced_cost'].strip('$')).strip(',').replace(",","")) if subnet_index['tag2'] in subnet_cost_dict.keys(): subnet_cost_dict[subnet_index['tag2']] += round(subnet_cost,2) else: subnet_cost_dict[subnet_index['tag2']] = round(subnet_cost,2) return subnet_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnetwise_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_current_prev_subnet_costs(self, start_date, end_date): #query parameters should be moved to config file subnet_cost_dict = {} try: previous_period = self.get_previous_period(start_date,end_date) previous_start_date, previous_end_date = previous_period[0], previous_period[1] subnet_cost_dict['current_period'] = self.get_subnet_costs(start_date, end_date) subnet_cost_dict['previous_period'] = self.get_subnet_costs(previous_start_date, previous_end_date) return subnet_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def split_string(self, input_string, delimiters): #this function splits a string on multiple delimiters delimiters = tuple(delimiters) string_list = [input_string,] for delimiter in delimiters: for index1, input_sub_string in enumerate(string_list): temp_var = input_sub_string.split(delimiter) string_list.pop(index1) for index2, input_sub_string in enumerate(temp_var): string_list.insert(index1+index2, input_sub_string) return string_list def get_ebs_costs(self, start_date, end_date): try: query_parameters = "&start_date="+start_date+"&end_date="+end_date+"&filters=usage_type=@EBS&dimensions=usage_type,tag1,&metrics=invoiced_cost&order=desc&auth_token=" ebs_cost_query = self.construct_cost_query(query_parameters) ebs_cost_json = self.generate_report(ebs_cost_query) ebs_details = ebs_cost_json['results'] ebs_cost_dict = collections.defaultdict(dict) for instance_index in ebs_details: if instance_index['tag1'] in ebs_cost_dict: ebs_cost_dict[instance_index['tag1']] += round(float(instance_index['invoiced_cost'].strip('$').replace(',','')),2) else: ebs_cost_dict[instance_index['tag1']] = round(float(instance_index['invoiced_cost'].strip('$').replace(',','')),2) return dict(ebs_cost_dict) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_instances_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_instances_costs(self, start_date, end_date): try: query_parameters = "&start_date="+start_date+"&end_date="+end_date+"&filters=service_key==AmazonEC2&dimensions=tag1,tag3,&metrics=invoiced_cost&order=desc&auth_token=" instance_cost_query = self.construct_cost_query(query_parameters) instance_cost_json = self.generate_report(instance_cost_query) instance_details = instance_cost_json['results'] instance_cost_dict = collections.defaultdict(dict) for instance_index in instance_details: if 'tag3' in instance_index: if instance_index['tag3'] in instance_cost_dict: instance_cost_dict[instance_index['tag3']]+= round(float(instance_index['invoiced_cost'].strip('$').replace(',','')),2) else: instance_cost_dict[instance_index['tag3']] = round(float(instance_index['invoiced_cost'].strip('$').replace(',','')),2) elif 'tag1' in instance_index: if instance_index['tag1'] in instance_cost_dict: instance_cost_dict[instance_index['tag1']] += round(float(instance_index['invoiced_cost'].strip('$').replace(',','')),2) else: instance_cost_dict[instance_index['tag1']] = round(float(instance_index['invoiced_cost'].strip('$').replace(',','')),2) return dict(instance_cost_dict) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_instances_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_current_prev_instances_costs(self, start_date, end_date): try: instance_cost_dict = {} previous_period = self.get_previous_period(start_date,end_date) previous_start_date, previous_end_date = previous_period[0], previous_period[1] instance_cost_dict['current_period'] = self.get_instances_costs(start_date, end_date) instance_cost_dict['previous_period'] = self.get_instances_costs(previous_start_date, previous_end_date) return instance_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return def get_percentage_change(self, cost_dict): percentage_dict, current_costs_dict, previous_costs_dict = {},{},{} try: if cost_dict: if cost_dict.has_key('current_period'): current_costs_dict = cost_dict['current_period'] if cost_dict.has_key('previous_period'): previous_costs_dict = cost_dict['previous_period'] else: raise Exception("Invalid value: No values for current and previous costs") for key in current_costs_dict: if key in previous_costs_dict.keys(): current_cost = current_costs_dict[key] previous_cost = previous_costs_dict[key] difference = current_cost - previous_cost if difference < 0: tag = 'decrease' elif difference >0: tag = 'increase' else: tag = 'equal' if previous_cost == 0.0: percentage = round((abs(difference)),2) else: percentage = round((abs(difference)*100/previous_cost),2) percentage_dict[key] = (current_cost, tag, percentage) else: percentage_dict[key] = (current_costs_dict[key], '',0) return percentage_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_cloudability_costs(self): cloud_cost_dict = self.memcache_var.get('cloud_costs') if cloud_cost_dict is None: cloud_cost_dict = self.memcache_var.get('global_cloudability_costs') if cloud_cost_dict is not None: self.memcache_var.set("cloud_costs", cloud_cost_dict, 600) with threading.RLock(): thread = threading.Thread(target= self.cache_cloud_costs) thread.start() return cloud_cost_dict def cache_cloud_costs(self): try: cloudability_dict = self.get_cloud_costs() self.memcache_var.set("cloud_costs", cloudability_dict,2*60*60) if cloudability_dict is None: raise Exception("Clodability data is not available. Please ensure data is available and populate the cache.") if cloudability_dict is not None: self.memcache_var.set("global_cloudability_costs", cloudability_dict,86400) self.memcache_var.disconnect_all() except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "cache_cloud_costs()", exp_object, exc_type, exc_obj, exc_tb) self.memcache_var.disconnect_all() def get_cloud_costs(self): current_date= date.today().strftime('%Y-%m-%d') month = date.today().month if month in [1, 3, 5, 7, 8, 10, 12]: previous_date = (date.today()-timedelta(days=30)).strftime('%Y-%m-%d') elif month in [2]: previous_date = (date.today()-timedelta(days=28)).strftime('%Y-%m-%d') else: previous_date = (date.today()-timedelta(days=29)).strftime('%Y-%m-%d') organization_list = self.aws_helper_object.get_organizations() region_list = self.aws_helper_object.get_regions() self.cloudability_dict = self.ah_obj.create_nested_defaultdict() for organization in organization_list: for region in region_list: vpc_list = self.aws_helper_object.get_vpc_in_region(region) if vpc_list: for vpc in ["ame1"]: if vpc: ec2_costs = self.get_ec2_costs(previous_date, current_date) self.cloudability_dict = self.ah_obj.create_nested_defaultdict() self.cloudability_dict['ec2_costs'][organization] = ec2_costs ec2_costs = self.get_current_prev_ec2_costs(previous_date, current_date) ec2_percentage_change = self.get_percentage_change(ec2_costs) self.cloudability_dict['ec2_percentage_change'][organization]= ec2_percentage_change environment_costs = self.get_environment_costs(previous_date, current_date) self.cloudability_dict['environment_costs'][organization] = environment_costs environment_costs = self.get_current_prev_environment_costs(previous_date, current_date) env_percentage_change = self.get_percentage_change(environment_costs) self.cloudability_dict['env_percentage_change'][organization] = env_percentage_change subnet_costs = subnet_costs = self.get_subnet_costs(previous_date, current_date) self.cloudability_dict['subnet_costs'][organization]= subnet_costs subnet_costs = self.get_current_prev_subnet_costs(previous_date, current_date) subnet_percentage_change = self.get_percentage_change(subnet_costs) self.cloudability_dict['subnet_percentage_change'][organization] = subnet_percentage_change instances_costs = self.get_instances_costs(previous_date, current_date) self.cloudability_dict['instances_costs'][organization]= instances_costs instances_costs = self.get_current_prev_instances_costs(previous_date, current_date) instances_percentage_change = self.get_percentage_change(instances_costs) self.cloudability_dict['instances_percentage_change'][organization] = instances_percentage_change ebs_costs = self.get_ebs_costs(previous_date, current_date) self.cloudability_dict['ebs_costs'][organization]= ebs_costs return self.ah_obj.defaultdict_to_dict(self.cloudability_dict)
class CloudabilityModule(AtlasBase): def __init__(self, request=None,environment=None): self.module = "cloudability_module" self.ah_obj = AtlasHelper() self.cloud_obj = Cloudability() self.awshelper_obj = AwsHelper() self.request = request self.aws_object = AwsModule(request, environment) self.instance_cost = 0.0 self.storage_cost = 0.0 def get_information(self, environment=None, **kwargs): organization_list = self.awshelper_obj.get_organizations() if environment is None: if 'env_cost_dict' in kwargs: if kwargs['env_cost_dict'] == 'true': for organization in organization_list: env_cost_dict= self.cloud_obj.get_cloudability_costs()['environment_costs'][organization] env_cost_dict['all'] = self.get_information(ec2_cost_dict='true')['region_zone'] return env_cost_dict if 'ec2_cost_dict' in kwargs: if kwargs['ec2_cost_dict'] == 'true': for organization in organization_list: ec2_costs= self.cloud_obj.get_cloudability_costs()['ec2_costs'][organization] return ec2_costs else: if 'env_cost_dict' in kwargs: if kwargs['env_cost_dict'] == 'true': env_costs = 0 for organization in organization_list: env_cost_dict = self.cloud_obj.get_cloudability_costs()['environment_costs'][organization] environment_groups = self.ah_obj.get_atlas_config_data("global_config_data", "environment_groups") if environment_groups and environment in environment_groups[1].keys(): if environment == 'all': env_group_for_environment = self.awshelper_obj.get_environments(organization) else: env_group_for_environment = environment_groups[1][environment] for env in env_group_for_environment: env_costs+=env_cost_dict[env] env_cost_dict[environment] = env_costs return env_cost_dict if 'apps_in_environment' in kwargs: if kwargs['apps_in_environment'] == 'true': return self.aws_object.get_information(environment, apps_in_environment='true') if 'instance_data' in kwargs: if kwargs['instance_data'] == 'true': return self.aws_object.get_information(environment, instance_data='true') if 'instances_cost_dict' in kwargs: if kwargs['instances_cost_dict'] == 'true': for organization in organization_list: return self.cloud_obj.get_cloudability_costs()['instances_costs'][organization] if 'ebs_cost_dict' in kwargs: if kwargs['ebs_cost_dict'] == 'true': for organization in organization_list: return self.cloud_obj.get_cloudability_costs()['ebs_costs'][organization] if 'aws_info_dict' in kwargs: if kwargs['aws_info_dict'] == 'true': return self.aws_object.get_information(environment, 'aws_info_dict') if 'application_subnets' in kwargs: if kwargs['application_subnets'] == 'true': return self.aws_object.get_information(environment, 'application_subnets') def get_configuration_data(self, key): value = self.ah_obj.get_atlas_config_data(self.module, key) if isinstance(value, dict): return value[0] else: return value def get_stack_attributes(self, environment=None): """ Get stack attributes from config file. """ stack_attribute_list = [] stack_attributes_dict = self.ah_obj.get_atlas_config_data('cloudability_module', 'stack_attributes')[1] for attribute, details in stack_attributes_dict.iteritems(): stack_attribute_list.append((attribute, details['editable'])) return(stack_attribute_list, stack_attributes_dict) def get_attribute_values(self, environment=None): return self.__get_detailed_instances_cost_dict(environment, 'stack_costs') def get_status(self,environment=None): status_information = self.get_configuration_data('status') cloud_status_dict = {} organization_list = self.awshelper_obj.get_organizations() environment_list = [] if environment==None: env_cost_dict = self.get_information(env_cost_dict='true') cloud_status_dict = {environment: ["$"+str(env_cost_dict[environment])] for environment in env_cost_dict.keys()} else: env_cost_dict = self.get_information(environment, env_cost_dict='true') region_vpc_selection = self.aws_object.get_information(environment, region_vpc_dict='true') if environment == "uncategorized": region_list = self.awshelper_obj.get_regions() for region in region_vpc_selection: if region =='east': cloud_status_dict[region] = ["$"+str(env_cost_dict[environment])] else: cloud_status_dict[region] = ["$"+ "0.0"] else: for vpc in ['ame1']: #should be changed later to include all vpcs. cloud_status_dict[vpc] = ["$"+str(env_cost_dict[environment])] return (status_information, cloud_status_dict) def get_tabs(self, environment=None): pass def get_instance_actions(self, environment=None): pass def get_environment_actions(self, environment=None): pass def get_instance_group_actions(self, environment=None): pass def get_stack_actions(self, environment=None): pass def get_vpc_actions(self): pass def get_action_status(self, json_data, environment=None): pass def perform_instance_actions(self, environment=None): pass def perform_instancegroup_actions(): pass def perform_stack_actions(): pass def perform_vpc_actions(self, json_data): pass def perform_environment_actions(self, environment=None): pass def get_columns(self, environment=None): column_list = self.ah_obj.get_atlas_config_data(self.module, 'columns') column_dict= self.ah_obj.create_nested_defaultdict() if column_list: column_dict = self.__get_detailed_instances_cost_dict(environment, 'instances_cost') return (column_list, self.ah_obj.defaultdict_to_dict(column_dict)) def get_action_parameters(self, action_type, environment=None): pass def load_session(self, request, environment=None): pass def save_session(self, request, environment=None): pass def get_defaults(): pass def get_aggregates(self, environment=None): aggregates = self.ah_obj.get_atlas_config_data(self.module, 'aggregates') if environment is None: aggregate_list = ["$"+str(self.get_information(ec2_cost_dict='true')['region_zone'])] return (aggregates, aggregate_list) else: aggregate_dict = collections.defaultdict(dict) for agg_key in aggregates: if agg_key == 'cost': aggregate_dict[agg_key] = self.get_information(env_cost_dict='true')[environment] return dict(aggregate_dict) def refresh_information(self, environment=None): self.cloud_obj.cache_cloud_costs() return def __get_detailed_instances_cost_dict(self, environment, cost_type): instances_cost_dict = self.get_information(environment, instances_cost_dict = 'true') ebs_cost_dict = self.get_information(environment, ebs_cost_dict = 'true') aws_tabs_dict = self.aws_object.get_tabs(environment)[1] instance_information = self.aws_object.get_information(environment, instance_data='true') organization_list = self.awshelper_obj.get_organizations() (stack_attr_list, stack_attr_details) = self.get_configuration_data('stack_attributes') apps_in_environment = self.get_information(environment, apps_in_environment='true') application_subnets = self.get_information(environment, application_subnets='true') region, vpc, subnet = "", "", "" instance_cost_column_dict= self.ah_obj.create_nested_defaultdict() ebs_cost_column_dict = self.ah_obj.create_nested_defaultdict() stack_cost_dict= self.ah_obj.create_nested_defaultdict() stack_cost_string_dict = self.ah_obj.create_nested_defaultdict() name_tag_value, fqdn_tag_value = "", "" for instance, aws_tabs_dict in aws_tabs_dict.iteritems(): attribute_cost={} if 'Name' in aws_tabs_dict['aws_tags']: name_tag_value = aws_tabs_dict['aws_tags']["Name"] if 'fqdn' in aws_tabs_dict['aws_tags']: fqdn_tag_value = aws_tabs_dict['aws_tags']['fqdn'] instance_details = self.ah_obj.get_nested_attribute_values(instance_information, instance)[1] region = instance_details['region'] if "region" in instance_details else "none" subnet = instance_details['subnet'] if "subnet" in instance_details else "none" attribute_cost[subnet] = {} vpc = instance_details['vpc'] if "vpc" in instance_details else "none" attribute_cost[vpc] = {} attribute_cost[vpc][subnet] = {} stack = instance_details['application'] if "application" in instance_details else "none" if cost_type == 'instances_cost' or cost_type == 'stack_costs': if fqdn_tag_value in ebs_cost_dict: self.storage_cost = ebs_cost_dict[fqdn_tag_value] if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['storage_cost'] = "$" +str(ebs_cost_dict[fqdn_tag_value])+"/m" else: instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['storage_cost'] = "$" +str(ebs_cost_dict[fqdn_tag_value])+"/m" else: self.storage_cost = 0.0 if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['storage_cost'] = "$" + "0.0"+"/m" else: instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['storage_cost'] = "$" + "0.0"+"/m" if fqdn_tag_value in instances_cost_dict: if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[fqdn_tag_value])+"/m" else: self.instance_cost = instances_cost_dict[fqdn_tag_value] instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[fqdn_tag_value])+"/m" elif name_tag_value in instances_cost_dict: self.instance_cost = instances_cost_dict[name_tag_value] if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[name_tag_value]) +"/m" else: self.instance_cost = instances_cost_dict[fqdn_tag_value] instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[name_tag_value]) +"/m" else: if environment=="uncategorized": instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['instance_cost'] = "(empty)" else: self.instance_cost = 0.0 instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['instance_cost'] = "(empty)" if cost_type == 'stack_costs': attr_cost = 0.0 for attribute in stack_attr_list: if attribute == 'instance_cost': attr_cost = self.instance_cost elif attribute == 'storage_cost': attr_cost = self.storage_cost elif attribute == 'total_cost': attr_cost = self.instance_cost + self.storage_cost if stack_attr_details[attribute]['stack'] == ['all']: for apps in apps_in_environment: if stack == apps: if not stack_cost_dict[region][vpc][subnet][stack][attribute]: stack_cost_dict[region][vpc][subnet][stack][attribute] = attr_cost else: cost = stack_cost_dict[region][vpc][subnet][stack][attribute] stack_cost_dict[region][vpc][subnet][stack][attribute]+=attr_cost else: for attr_stack in stack_attr_details[attribute]['stack']: if attr_stack == stack: if not stack_cost_dict[region][vpc][subnet][stack][attribute]: stack_cost_dict[region][vpc][subnet][stack][attribute] = attr_cost else: stack_cost_dict[region][vpc][subnet][stack][attribute]+=attr_cost stack_cost_string_dict[region][vpc][subnet][stack][attribute] = \ "$"+str(stack_cost_dict[region][vpc][subnet][stack][attribute])+"/m" if cost_type == 'stack_costs': return self.ah_obj.defaultdict_to_dict(stack_cost_string_dict) if cost_type == 'instances_cost' : return self.ah_obj.defaultdict_to_dict(instance_cost_column_dict)
class AwsInstanceHelper: #class to fetch data from aws for atlas def __init__(self, environment=None): """Initialize variables.""" self.ah_obj = AtlasHelper() self.awshelper_obj = AwsHelper() self.module = 'aws_module' self.environment_dict = {} self.env_subnet_dict = {} self.env_details_dict = {} self.environment_groups = self.ah_obj.get_atlas_config_data( "global_config_data", "environment_groups") def get_aws_details(self, regionvpcselected): organizations = self.awshelper_obj.get_organizations() aws_details_dict = {} environment_list = [] aws_details = self.awshelper_obj.get_instances_details() for organization in organizations: environment_list = self.awshelper_obj.get_environments( organization) environment_list.append('uncategorized') environment_list.append('all') ah_obj = AtlasHelper() total_count, total_running, total_stopped = 0, 0, 0 region_dict = {} region_env_list = [] for environment in environment_list: if environment != 'all': env_attributes = self.ah_obj.get_nested_attribute_values( aws_details, environment)[1] for region in regionvpcselected.keys(): if 'regions' in env_attributes.keys( ) and region in env_attributes['regions'].keys(): region_keys, region_attributes = ah_obj.get_nested_attribute_values( env_attributes['regions'], region) if 'count' in region_keys and 'running' in region_keys and 'stopped' in region_keys: region_count = region_attributes['count'] region_running = region_attributes['running'] region_stopped = region_attributes['stopped'] if environment == "uncategorized": total_count += region_count total_running += region_running total_stopped += region_stopped region_env_list.append([ region, "none", environment, region_count, region_running, region_stopped ]) else: vpc_list = regionvpcselected[region] for vpc in vpc_list: if 'vpc' in region_attributes.keys(): vpc_attributes = self.ah_obj.get_nested_attribute_values( region_attributes['vpc'], vpc)[1] vpc_count = vpc_attributes['count'] vpc_running = vpc_attributes[ 'running'] vpc_stopped = vpc_attributes[ 'stopped'] total_count += vpc_count total_running += vpc_running total_stopped += vpc_stopped region_env_list.append([ region, vpc, environment, vpc_count, vpc_running, vpc_stopped ]) else: region_env_list.append([ region, vpc, environment, total_count, total_running, total_stopped ]) temp_list, total_list = [], [] for count in range(0, len(environment_list)): total_list.append([environment_list[count], 0, 0, 0]) for index in range(0, len(region_env_list)): for index1 in range(0, len(total_list)): if region_env_list[index][2] == total_list[index1][0]: temp_list = [ region_env_list[index][3], region_env_list[index][4], region_env_list[index][5] ] total_list[index1][1] += temp_list[0] total_list[index1][2] += temp_list[1] total_list[index1][3] += temp_list[2] aws_details_dict['total_count'] = total_count aws_details_dict['total_running'] = total_running aws_details_dict['total_stopped'] = total_stopped aws_details_dict['environment_list'] = environment_list aws_details_dict['region_dict'] = region_dict aws_details_dict['total_list'] = total_list return aws_details_dict def calculate_environment_aggregates(self, environment, env_attributes, region_vpc_dict): """ Calculate aggregate values for each environment. """ total_list = [0, 0, 0] for region in region_vpc_dict.keys(): if region in env_attributes['regions'].keys(): region_attributes = env_attributes['regions'][region] if environment == 'uncategorized': region_aggs = [ region_attributes['count'], region_attributes['running'], region_attributes['stopped'] ] total_list = map(add, total_list, region_aggs) else: for vpc in region_vpc_dict[region]: if vpc and vpc in region_attributes['vpc'].keys(): vpc_properties = self.ah_obj.get_nested_attribute_values( env_attributes['regions'][region], vpc)[1] vpc_aggregates = [ vpc_properties['count'], vpc_properties['running'], vpc_properties['stopped'] ] total_list = map(add, total_list, vpc_aggregates) return total_list def get_environment_aggregates(self, environment, region_vpc_dict): """ Get aggregate status values. """ aggregate_list = [0, 0, 0] organizations = self.awshelper_obj.get_organizations() org_attributes = self.ah_obj.get_nested_attribute_values( self.awshelper_obj.get_instances_details(), 'organizations')[1] for organization in organizations: if self.environment_groups and environment in self.environment_groups[ 1].keys(): if environment == 'all': env_group_for_environment = self.awshelper_obj.get_environments( organization) else: env_group_for_environment = self.environment_groups[1][ environment] for env in env_group_for_environment: env_attributes = self.ah_obj.get_nested_attribute_values( org_attributes[organization], env)[1] env_agg_list = self.calculate_environment_aggregates( env, env_attributes, region_vpc_dict) aggregate_list = map(add, aggregate_list, env_agg_list) else: env_attributes = self.ah_obj.get_nested_attribute_values( org_attributes[organization], environment)[1] aggregate_list = self.calculate_environment_aggregates( environment, env_attributes, region_vpc_dict) return aggregate_list def determine_environment_status(self, environment, env_attributes, region_vpc_dict): vpc_status_dict = {} for region in region_vpc_dict.keys(): if region not in env_attributes['regions'].keys(): continue else: region_attributes = env_attributes['regions'][region] vpc_status_dict[region] = {'status': []} if environment == 'uncategorized': vpc_status_dict[region]['status'] = [ region_attributes['count'], region_attributes['running'], region_attributes['stopped'] ] else: for vpc in region_vpc_dict[region]: if vpc and vpc in region_attributes['vpc'].keys(): vpc_status_dict[vpc] = {'status': []} vpc_properties = self.ah_obj.get_nested_attribute_values( env_attributes['regions'][region], vpc)[1] vpc_status_dict[vpc]['status'] = [ vpc_properties['count'], vpc_properties['running'], vpc_properties['stopped'] ] return vpc_status_dict def get_environment_status(self, environment, region_vpc_dict): """Get environment status.""" ah_obj = AtlasHelper() vpc_status_dict = {} organizations = self.awshelper_obj.get_organizations() org_attributes = self.ah_obj.get_nested_attribute_values( self.awshelper_obj.get_instances_details(), 'organizations')[1] for organization in organizations: if self.environment_groups and environment in self.environment_groups[ 1].keys(): env_status_dict = {} if environment == 'all': env_group_for_env = self.awshelper_obj.get_environments( organization) else: env_group_for_env = self.environment_groups[1][environment] for env in env_group_for_env: env_attributes = self.ah_obj.get_nested_attribute_values( org_attributes[organization], env)[1] env_status_dict = self.determine_environment_status( env, env_attributes, region_vpc_dict) if not vpc_status_dict: vpc_status_dict.update(env_status_dict) else: for vpc in env_status_dict.keys(): vpc_status_dict[vpc]['status'] = map( add, vpc_status_dict[vpc]['status'], env_status_dict[vpc]['status']) else: env_attributes = self.ah_obj.get_nested_attribute_values( org_attributes[organization], environment)[1] vpc_status_dict = self.determine_environment_status( environment, env_attributes, region_vpc_dict) return vpc_status_dict def find_env_details(self, environment, env_attributes, region_vpc_dict): organizations = self.awshelper_obj.get_organizations() details_dict, env_subnets_dict = {}, {} environment_subnets = {'vpc': {}} apps_in_environment, instances_list, application_subnets, env_subnet_list = [], [], [], [] vpc_attribute_dict, aws_info_dict, env_info_dict = {}, {}, {} instances_list, apps_in_environment, env_subnet_list = [], [], [] subnets_with_instances = {} #new addition instance_keys = self.ah_obj.get_atlas_config_data( self.module, "instance_keys") for region in region_vpc_dict.keys(): if region not in env_attributes['regions'].keys(): continue else: region_attributes = env_attributes['regions'][region] if environment == 'uncategorized': vpc_attribute_dict[region] = { "subnets": { "none": { "instance_attributes": {} } } } for instance, details in region_attributes[ 'uncat_instances'].iteritems(): if details.has_key('instance_attributes') and details[ 'instance_attributes'].has_key('instance_id'): instance_id = details['instance_attributes'][ 'instance_id'] vpc_attribute_dict[region]["subnets"]["none"][ "instance_attributes"][instance_id] = details[ 'instance_attributes'] aws_info_dict[instance_id] = {} aws_info_dict[instance_id][ 'aws_information'] = details['aws_information'] aws_info_dict[instance_id]['aws_tags'] = details[ 'aws_tags'] else: for vpc in region_vpc_dict[region]: if vpc and vpc in region_attributes['vpc'].keys(): environment_subnets['vpc'][vpc] = {'subnets': []} env_subnet_list.extend( self.awshelper_obj.get_subnets_in_environment( region, vpc, environment)) environment_subnets['vpc'][vpc][ 'subnets'] = env_subnet_list vpc_attribute_dict[vpc] = { 'subnets': {}, 'applications': [] } vpc_attribute_dict[vpc]['subnets'] = {} vpc_properties = self.ah_obj.get_nested_attribute_values( env_attributes['regions'][region], vpc)[1] if 'subnets' in vpc_properties.keys(): for subnet in vpc_properties['subnets'].keys(): instances_details = vpc_properties[ 'subnets'][subnet]['instances'] if instances_details: #env_subnet_list.append(subnet) subnets_with_instances[subnet] = [] vpc_attribute_dict[vpc]['subnets'][ subnet] = { 'instance_attributes': {} } if subnet not in application_subnets: application_subnets.append(subnet) for instance, details in instances_details.iteritems( ): instance_attributes = details[ 'instance_attributes'] if instance_attributes: details_key = details[ 'instance_attributes'][ 'instance_id'] vpc_attribute_dict[vpc][ 'subnets'][subnet][ 'instance_attributes'][ details_key] = instance_attributes aws_info_dict[details_key] = {} aws_info_dict[details_key][ 'aws_information'] = details[ 'aws_information'] aws_info_dict[details_key][ 'aws_tags'] = details[ 'aws_tags'] instance_application = instance_attributes[ 'application'] if instance_application not in subnets_with_instances[ subnet]: subnets_with_instances[ subnet].append( instance_application) if instance_attributes[ 'application'] not in apps_in_environment: apps_in_environment.append( instance_attributes[ 'application']) details_dict['subnets_with_instances'] = subnets_with_instances details_dict['environment_subnets'] = environment_subnets details_dict['application_subnets'] = application_subnets details_dict['vpc_attribute_dict'] = vpc_attribute_dict details_dict['aws_info_dict'] = aws_info_dict details_dict['env_subnet_list'] = env_subnet_list details_dict['apps_in_environment'] = apps_in_environment return details_dict def get_environment_details(self, environment, region_vpc_dict): """ Get details for a particular environment or environment_group. """ organizations = self.awshelper_obj.get_organizations() org_attributes = self.ah_obj.get_nested_attribute_values( self.awshelper_obj.get_instances_details(), 'organizations')[1] for organization in organizations: if self.environment_groups and environment in self.environment_groups[ 0]: if environment == 'all': env_group_for_environment = self.awshelper_obj.get_environments( organization) else: env_group_for_environment = self.environment_groups[1][ environment] for env in env_group_for_environment: env_attributes = self.ah_obj.get_nested_attribute_values( org_attributes[organization], env)[1] environment_details = self.find_env_details( env, env_attributes, region_vpc_dict) if not self.env_details_dict: self.env_details_dict.update(environment_details) else: generator_obj = self.ah_obj.merge_dictionaries( self.env_details_dict, environment_details) self.env_details_dict = { key: value for key, value in generator_obj } else: env_attributes = self.ah_obj.get_nested_attribute_values( org_attributes[organization], environment)[1] self.env_details_dict = self.find_env_details( environment, env_attributes, region_vpc_dict) return self.env_details_dict def get_column_data(self, environment): column_dict = {} if 'vpc_attribute_dict' in self.env_details_dict: environment_attributes = self.env_details_dict[ 'vpc_attribute_dict'] for vpc, vpc_attrs_dict in environment_attributes.iteritems(): column_dict[vpc] = {'subnets': {}} if 'subnets' in vpc_attrs_dict: for subnet, subnet_attributes in vpc_attrs_dict[ 'subnets'].iteritems(): column_dict[vpc]['subnets'][subnet] = { 'instance_attributes': [] } if 'instance_attributes' in subnet_attributes: column_dict[vpc]['subnets'][subnet][ 'instance_attributes'] = subnet_attributes[ 'instance_attributes'] return column_dict
class Cloudability: #constructor def __init__(self): self.cloudability_dict = {} self.ah_obj = AtlasHelper() self.aws_helper_object = AwsHelper() self.module = "cloudability_module" self.auth_token = os.environ.get('CLOUDABILITY_AUTH_TOKEN') self.cl_base_url = self.ah_obj.get_atlas_config_data( self.module, "cloudability_base_url") self.cl_cost_url = self.ah_obj.get_atlas_config_data( self.module, "cloudability_cost_url") self.report_query = "" self.memcache_var = memcache.Client([ self.ah_obj.get_atlas_config_data("global_config_data", 'memcache_server_location') ], debug=1) self.environment_subnets_details = self.aws_helper_object.get_environment_subnets_details( ) def construct_cost_query(self, query_parameters): try: self.report_query = self.cl_base_url + self.cl_cost_url + query_parameters + self.auth_token return self.report_query except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "construct_cost_query()", exp_object, exc_type, exc_obj, exc_tb) return def generate_report(self, query): try: report_json = {} response = requests.get(query) if response.status_code == 200: report_json = json.loads( response.text) #convert the json into a python dictionary return report_json except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_previous_period(self, start_date, end_date): try: start = datetime.datetime.strptime(start_date, '%Y-%m-%d') end = datetime.datetime.strptime(end_date, '%Y-%m-%d') period = ((end - start).days) + 1 previous_start_date = ( start - datetime.timedelta(days=period)).strftime('%Y-%m-%d') previous_end_date = ( end - datetime.timedelta(days=period)).strftime('%Y-%m-%d') return (previous_start_date, previous_end_date) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_previous_period()", exp_object, exc_type, exc_obj, exc_tb) return #get ec2 costs def get_ec2_costs(self, start_date, end_date): ec2_costs = {'region_zone': 0.0} try: query_parameters = "verbose=1&start_date=" + start_date + "&end_date=" + end_date + "&dimensions=linked_account_name&metrics=invoiced_cost&sort_by=invoiced_cost&order=desc&max_results=50&offset=0&auth_token=" ec2_costs_query = self.construct_cost_query(query_parameters) ec2_cost_dict = self.generate_report(ec2_costs_query) if ec2_cost_dict: ec2_costs['region_zone'] = round( float(ec2_cost_dict['meta']['aggregates'][0] ['value'].strip('$').replace(',', '')), 2) if ec2_costs['region_zone']: return ec2_costs else: raise Exception("Could not generate EC2 costs") except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "calculate_ec2_costs()", exp_object, exc_type, exc_obj, exc_tb) return ec2_costs #get aggregate cost spent for ec2 for the specified period def get_current_prev_ec2_costs(self, start_date, end_date): #query parameters should be moved to config file ec2_cost_dict = {'current_period': 0.0, 'previous_period': 0.0} try: previous_period = self.get_previous_period(start_date, end_date) previous_start_date, previous_end_date = previous_period[ 0], previous_period[1] ec2_cost_dict['current_period'] = self.get_ec2_costs( start_date, end_date) ec2_cost_dict['previous_period'] = self.get_ec2_costs( previous_start_date, previous_end_date) return ec2_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_ec2_costs()", exp_object, exc_type, exc_obj, exc_tb) return def create_envcost_dict(self): try: env_subnet_zip = self.environment_subnets_details per_environment_costs = {} for env_subnet_tuple in env_subnet_zip: per_environment_costs[env_subnet_tuple[0]] = 0 return per_environment_costs except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "create_envcost_dict()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_environment_costs(self, start_date, end_date): try: query_parameters = "verbose=1&start_date=" + start_date + "&end_date=" + end_date + "&dimensions=tag2&metrics=invoiced_cost&sort_by=invoiced_cost&order=desc&auth_token=" subnets_cost_query = self.construct_cost_query(query_parameters) subnet_cost_dict = self.generate_report(subnets_cost_query) cost_dict = self.create_envcost_dict() env_subnet_zip = self.environment_subnets_details subnet_details = subnet_cost_dict['results'] for subnet_index in subnet_details: for env_subnet_tuple in env_subnet_zip: if subnet_index['tag2'] in env_subnet_tuple[1]: #strip of the $ symbol and , convert the string to float with 2 precisions subnet_cost = float( (subnet_index['invoiced_cost'].strip('$') ).strip(',').replace(",", "")) env_cost = cost_dict[env_subnet_tuple[0]] if subnet_index['tag2'] in cost_dict.keys(): cost_dict[env_subnet_tuple[0]] += round( (env_cost + subnet_cost), 2) else: cost_dict[env_subnet_tuple[0]] = round( (env_cost + subnet_cost), 2) if cost_dict: return cost_dict else: raise Exception('Could not calculate environment costs') except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_environment_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_current_prev_environment_costs(self, start_date, end_date): #query parameters should be moved to config file try: env_cost_dict = {} previous_period = self.get_previous_period(start_date, end_date) previous_start_date, previous_end_date = previous_period[ 0], previous_period[1] env_cost_dict['current_period'] = self.get_environment_costs( start_date, end_date) env_cost_dict['previous_period'] = self.get_environment_costs( previous_start_date, previous_end_date) return env_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return #calculate costs subnet wise def get_subnet_costs(self, start_date, end_date): subnet_cost_dict = {} try: query_parameters = "verbose=1&start_date=" + start_date + "&end_date=" + end_date + "&dimensions=tag2&metrics=invoiced_cost&sort_by=invoiced_cost&order=desc&auth_token=" subnets_cost_query = self.construct_cost_query(query_parameters) subnet_cost_json = self.generate_report(subnets_cost_query) env_subnet_zip = self.environment_subnets_details subnet_details = subnet_cost_json['results'] for subnet_index in subnet_details: for env_subnet_tuple in env_subnet_zip: if subnet_index['tag2'] in env_subnet_tuple[1]: #strip of the $ symbol and , convert the string to float with 2 precisions subnet_cost = float( (subnet_index['invoiced_cost'].strip('$') ).strip(',').replace(",", "")) if subnet_index['tag2'] in subnet_cost_dict.keys(): subnet_cost_dict[subnet_index['tag2']] += round( subnet_cost, 2) else: subnet_cost_dict[subnet_index['tag2']] = round( subnet_cost, 2) return subnet_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnetwise_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_current_prev_subnet_costs(self, start_date, end_date): #query parameters should be moved to config file subnet_cost_dict = {} try: previous_period = self.get_previous_period(start_date, end_date) previous_start_date, previous_end_date = previous_period[ 0], previous_period[1] subnet_cost_dict['current_period'] = self.get_subnet_costs( start_date, end_date) subnet_cost_dict['previous_period'] = self.get_subnet_costs( previous_start_date, previous_end_date) return subnet_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def split_string(self, input_string, delimiters): #this function splits a string on multiple delimiters delimiters = tuple(delimiters) string_list = [ input_string, ] for delimiter in delimiters: for index1, input_sub_string in enumerate(string_list): temp_var = input_sub_string.split(delimiter) string_list.pop(index1) for index2, input_sub_string in enumerate(temp_var): string_list.insert(index1 + index2, input_sub_string) return string_list def get_ebs_costs(self, start_date, end_date): try: query_parameters = "&start_date=" + start_date + "&end_date=" + end_date + "&filters=usage_type=@EBS&dimensions=usage_type,tag1,&metrics=invoiced_cost&order=desc&auth_token=" ebs_cost_query = self.construct_cost_query(query_parameters) ebs_cost_json = self.generate_report(ebs_cost_query) ebs_details = ebs_cost_json['results'] ebs_cost_dict = collections.defaultdict(dict) for instance_index in ebs_details: if instance_index['tag1'] in ebs_cost_dict: ebs_cost_dict[instance_index['tag1']] += round( float( instance_index['invoiced_cost'].strip('$').replace( ',', '')), 2) else: ebs_cost_dict[instance_index['tag1']] = round( float( instance_index['invoiced_cost'].strip('$').replace( ',', '')), 2) return dict(ebs_cost_dict) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_instances_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_instances_costs(self, start_date, end_date): try: query_parameters = "&start_date=" + start_date + "&end_date=" + end_date + "&filters=service_key==AmazonEC2&dimensions=tag1,tag3,&metrics=invoiced_cost&order=desc&auth_token=" instance_cost_query = self.construct_cost_query(query_parameters) instance_cost_json = self.generate_report(instance_cost_query) instance_details = instance_cost_json['results'] instance_cost_dict = collections.defaultdict(dict) for instance_index in instance_details: if 'tag3' in instance_index: if instance_index['tag3'] in instance_cost_dict: instance_cost_dict[instance_index['tag3']] += round( float(instance_index['invoiced_cost'].strip( '$').replace(',', '')), 2) else: instance_cost_dict[instance_index['tag3']] = round( float(instance_index['invoiced_cost'].strip( '$').replace(',', '')), 2) elif 'tag1' in instance_index: if instance_index['tag1'] in instance_cost_dict: instance_cost_dict[instance_index['tag1']] += round( float(instance_index['invoiced_cost'].strip( '$').replace(',', '')), 2) else: instance_cost_dict[instance_index['tag1']] = round( float(instance_index['invoiced_cost'].strip( '$').replace(',', '')), 2) return dict(instance_cost_dict) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_instances_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_current_prev_instances_costs(self, start_date, end_date): try: instance_cost_dict = {} previous_period = self.get_previous_period(start_date, end_date) previous_start_date, previous_end_date = previous_period[ 0], previous_period[1] instance_cost_dict['current_period'] = self.get_instances_costs( start_date, end_date) instance_cost_dict['previous_period'] = self.get_instances_costs( previous_start_date, previous_end_date) return instance_cost_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return def get_percentage_change(self, cost_dict): percentage_dict, current_costs_dict, previous_costs_dict = {}, {}, {} try: if cost_dict: if cost_dict.has_key('current_period'): current_costs_dict = cost_dict['current_period'] if cost_dict.has_key('previous_period'): previous_costs_dict = cost_dict['previous_period'] else: raise Exception( "Invalid value: No values for current and previous costs") for key in current_costs_dict: if key in previous_costs_dict.keys(): current_cost = current_costs_dict[key] previous_cost = previous_costs_dict[key] difference = current_cost - previous_cost if difference < 0: tag = 'decrease' elif difference > 0: tag = 'increase' else: tag = 'equal' if previous_cost == 0.0: percentage = round((abs(difference)), 2) else: percentage = round( (abs(difference) * 100 / previous_cost), 2) percentage_dict[key] = (current_cost, tag, percentage) else: percentage_dict[key] = (current_costs_dict[key], '', 0) return percentage_dict except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "get_subnet_costs()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_cloudability_costs(self): cloud_cost_dict = self.memcache_var.get('cloud_costs') if cloud_cost_dict is None: cloud_cost_dict = self.memcache_var.get( 'global_cloudability_costs') if cloud_cost_dict is not None: self.memcache_var.set("cloud_costs", cloud_cost_dict, 600) with threading.RLock(): thread = threading.Thread(target=self.cache_cloud_costs) thread.start() return cloud_cost_dict def cache_cloud_costs(self): try: cloudability_dict = self.get_cloud_costs() self.memcache_var.set("cloud_costs", cloudability_dict, 2 * 60 * 60) if cloudability_dict is None: raise Exception( "Clodability data is not available. Please ensure data is available and populate the cache." ) if cloudability_dict is not None: self.memcache_var.set("global_cloudability_costs", cloudability_dict, 86400) self.memcache_var.disconnect_all() except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("cloudability.py", "cache_cloud_costs()", exp_object, exc_type, exc_obj, exc_tb) self.memcache_var.disconnect_all() def get_cloud_costs(self): current_date = date.today().strftime('%Y-%m-%d') month = date.today().month if month in [1, 3, 5, 7, 8, 10, 12]: previous_date = (date.today() - timedelta(days=30)).strftime('%Y-%m-%d') elif month in [2]: previous_date = (date.today() - timedelta(days=28)).strftime('%Y-%m-%d') else: previous_date = (date.today() - timedelta(days=29)).strftime('%Y-%m-%d') organization_list = self.aws_helper_object.get_organizations() region_list = self.aws_helper_object.get_regions() self.cloudability_dict = self.ah_obj.create_nested_defaultdict() for organization in organization_list: for region in region_list: vpc_list = self.aws_helper_object.get_vpc_in_region(region) if vpc_list: for vpc in ["ame1"]: if vpc: ec2_costs = self.get_ec2_costs( previous_date, current_date) self.cloudability_dict = self.ah_obj.create_nested_defaultdict( ) self.cloudability_dict['ec2_costs'][ organization] = ec2_costs ec2_costs = self.get_current_prev_ec2_costs( previous_date, current_date) ec2_percentage_change = self.get_percentage_change( ec2_costs) self.cloudability_dict['ec2_percentage_change'][ organization] = ec2_percentage_change environment_costs = self.get_environment_costs( previous_date, current_date) self.cloudability_dict['environment_costs'][ organization] = environment_costs environment_costs = self.get_current_prev_environment_costs( previous_date, current_date) env_percentage_change = self.get_percentage_change( environment_costs) self.cloudability_dict['env_percentage_change'][ organization] = env_percentage_change subnet_costs = subnet_costs = self.get_subnet_costs( previous_date, current_date) self.cloudability_dict['subnet_costs'][ organization] = subnet_costs subnet_costs = self.get_current_prev_subnet_costs( previous_date, current_date) subnet_percentage_change = self.get_percentage_change( subnet_costs) self.cloudability_dict['subnet_percentage_change'][ organization] = subnet_percentage_change instances_costs = self.get_instances_costs( previous_date, current_date) self.cloudability_dict['instances_costs'][ organization] = instances_costs instances_costs = self.get_current_prev_instances_costs( previous_date, current_date) instances_percentage_change = self.get_percentage_change( instances_costs) self.cloudability_dict[ 'instances_percentage_change'][ organization] = instances_percentage_change ebs_costs = self.get_ebs_costs( previous_date, current_date) self.cloudability_dict['ebs_costs'][ organization] = ebs_costs return self.ah_obj.defaultdict_to_dict(self.cloudability_dict)
class CloudabilityModule(AtlasBase): def __init__(self, request=None, environment=None): self.module = "cloudability_module" self.ah_obj = AtlasHelper() self.cloud_obj = Cloudability() self.awshelper_obj = AwsHelper() self.request = request self.aws_object = AwsModule(request, environment) self.instance_cost = 0.0 self.storage_cost = 0.0 def get_information(self, environment=None, **kwargs): organization_list = self.awshelper_obj.get_organizations() if environment is None: if 'env_cost_dict' in kwargs: if kwargs['env_cost_dict'] == 'true': for organization in organization_list: env_cost_dict = self.cloud_obj.get_cloudability_costs( )['environment_costs'][organization] env_cost_dict['all'] = self.get_information( ec2_cost_dict='true')['region_zone'] return env_cost_dict if 'ec2_cost_dict' in kwargs: if kwargs['ec2_cost_dict'] == 'true': for organization in organization_list: ec2_costs = self.cloud_obj.get_cloudability_costs( )['ec2_costs'][organization] return ec2_costs else: if 'env_cost_dict' in kwargs: if kwargs['env_cost_dict'] == 'true': env_costs = 0 for organization in organization_list: env_cost_dict = self.cloud_obj.get_cloudability_costs( )['environment_costs'][organization] environment_groups = self.ah_obj.get_atlas_config_data( "global_config_data", "environment_groups") if environment_groups and environment in environment_groups[ 1].keys(): if environment == 'all': env_group_for_environment = self.awshelper_obj.get_environments( organization) else: env_group_for_environment = environment_groups[ 1][environment] for env in env_group_for_environment: env_costs += env_cost_dict[env] env_cost_dict[environment] = env_costs return env_cost_dict if 'apps_in_environment' in kwargs: if kwargs['apps_in_environment'] == 'true': return self.aws_object.get_information( environment, apps_in_environment='true') if 'instance_data' in kwargs: if kwargs['instance_data'] == 'true': return self.aws_object.get_information( environment, instance_data='true') if 'instances_cost_dict' in kwargs: if kwargs['instances_cost_dict'] == 'true': for organization in organization_list: return self.cloud_obj.get_cloudability_costs( )['instances_costs'][organization] if 'ebs_cost_dict' in kwargs: if kwargs['ebs_cost_dict'] == 'true': for organization in organization_list: return self.cloud_obj.get_cloudability_costs( )['ebs_costs'][organization] if 'aws_info_dict' in kwargs: if kwargs['aws_info_dict'] == 'true': return self.aws_object.get_information( environment, 'aws_info_dict') if 'application_subnets' in kwargs: if kwargs['application_subnets'] == 'true': return self.aws_object.get_information( environment, 'application_subnets') def get_configuration_data(self, key): value = self.ah_obj.get_atlas_config_data(self.module, key) if isinstance(value, dict): return value[0] else: return value def get_stack_attributes(self, environment=None): """ Get stack attributes from config file. """ stack_attribute_list = [] stack_attributes_dict = self.ah_obj.get_atlas_config_data( 'cloudability_module', 'stack_attributes')[1] for attribute, details in stack_attributes_dict.iteritems(): stack_attribute_list.append((attribute, details['editable'])) return (stack_attribute_list, stack_attributes_dict) def get_attribute_values(self, environment=None): return self.__get_detailed_instances_cost_dict(environment, 'stack_costs') def get_status(self, environment=None): status_information = self.get_configuration_data('status') cloud_status_dict = {} organization_list = self.awshelper_obj.get_organizations() environment_list = [] if environment == None: env_cost_dict = self.get_information(env_cost_dict='true') cloud_status_dict = { environment: ["$" + str(env_cost_dict[environment])] for environment in env_cost_dict.keys() } else: env_cost_dict = self.get_information(environment, env_cost_dict='true') region_vpc_selection = self.aws_object.get_information( environment, region_vpc_dict='true') if environment == "uncategorized": region_list = self.awshelper_obj.get_regions() for region in region_vpc_selection: if region == 'east': cloud_status_dict[region] = [ "$" + str(env_cost_dict[environment]) ] else: cloud_status_dict[region] = ["$" + "0.0"] else: for vpc in ['ame1' ]: #should be changed later to include all vpcs. cloud_status_dict[vpc] = [ "$" + str(env_cost_dict[environment]) ] return (status_information, cloud_status_dict) def get_tabs(self, environment=None): pass def get_instance_actions(self, environment=None): pass def get_environment_actions(self, environment=None): pass def get_instance_group_actions(self, environment=None): pass def get_stack_actions(self, environment=None): pass def get_vpc_actions(self): pass def get_action_status(self, json_data, environment=None): pass def perform_instance_actions(self, environment=None): pass def perform_instancegroup_actions(): pass def perform_stack_actions(): pass def perform_vpc_actions(self, json_data): pass def perform_environment_actions(self, environment=None): pass def get_columns(self, environment=None): column_list = self.ah_obj.get_atlas_config_data(self.module, 'columns') column_dict = self.ah_obj.create_nested_defaultdict() if column_list: column_dict = self.__get_detailed_instances_cost_dict( environment, 'instances_cost') return (column_list, self.ah_obj.defaultdict_to_dict(column_dict)) def get_action_parameters(self, action_type, environment=None): pass def load_session(self, request, environment=None): pass def save_session(self, request, environment=None): pass def get_defaults(): pass def get_aggregates(self, environment=None): aggregates = self.ah_obj.get_atlas_config_data(self.module, 'aggregates') if environment is None: aggregate_list = [ "$" + str(self.get_information(ec2_cost_dict='true')['region_zone']) ] return (aggregates, aggregate_list) else: aggregate_dict = collections.defaultdict(dict) for agg_key in aggregates: if agg_key == 'cost': aggregate_dict[agg_key] = self.get_information( env_cost_dict='true')[environment] return dict(aggregate_dict) def refresh_information(self, environment=None): self.cloud_obj.cache_cloud_costs() return def __get_detailed_instances_cost_dict(self, environment, cost_type): instances_cost_dict = self.get_information(environment, instances_cost_dict='true') ebs_cost_dict = self.get_information(environment, ebs_cost_dict='true') aws_tabs_dict = self.aws_object.get_tabs(environment)[1] instance_information = self.aws_object.get_information( environment, instance_data='true') organization_list = self.awshelper_obj.get_organizations() (stack_attr_list, stack_attr_details) = self.get_configuration_data('stack_attributes') apps_in_environment = self.get_information(environment, apps_in_environment='true') application_subnets = self.get_information(environment, application_subnets='true') region, vpc, subnet = "", "", "" instance_cost_column_dict = self.ah_obj.create_nested_defaultdict() ebs_cost_column_dict = self.ah_obj.create_nested_defaultdict() stack_cost_dict = self.ah_obj.create_nested_defaultdict() stack_cost_string_dict = self.ah_obj.create_nested_defaultdict() name_tag_value, fqdn_tag_value = "", "" for instance, aws_tabs_dict in aws_tabs_dict.iteritems(): attribute_cost = {} if 'Name' in aws_tabs_dict['aws_tags']: name_tag_value = aws_tabs_dict['aws_tags']["Name"] if 'fqdn' in aws_tabs_dict['aws_tags']: fqdn_tag_value = aws_tabs_dict['aws_tags']['fqdn'] instance_details = self.ah_obj.get_nested_attribute_values( instance_information, instance)[1] region = instance_details[ 'region'] if "region" in instance_details else "none" subnet = instance_details[ 'subnet'] if "subnet" in instance_details else "none" attribute_cost[subnet] = {} vpc = instance_details[ 'vpc'] if "vpc" in instance_details else "none" attribute_cost[vpc] = {} attribute_cost[vpc][subnet] = {} stack = instance_details[ 'application'] if "application" in instance_details else "none" if cost_type == 'instances_cost' or cost_type == 'stack_costs': if fqdn_tag_value in ebs_cost_dict: self.storage_cost = ebs_cost_dict[fqdn_tag_value] if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none'][ 'instance_attributes'][instance][ 'storage_cost'] = "$" + str( ebs_cost_dict[fqdn_tag_value]) + "/m" else: instance_cost_column_dict[vpc]['subnets'][subnet][ 'instance_attributes'][instance][ 'storage_cost'] = "$" + str( ebs_cost_dict[fqdn_tag_value]) + "/m" else: self.storage_cost = 0.0 if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none'][ 'instance_attributes'][instance][ 'storage_cost'] = "$" + "0.0" + "/m" else: instance_cost_column_dict[vpc]['subnets'][subnet][ 'instance_attributes'][instance][ 'storage_cost'] = "$" + "0.0" + "/m" if fqdn_tag_value in instances_cost_dict: if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none'][ 'instance_attributes'][instance][ 'instance_cost'] = "$" + str( instances_cost_dict[fqdn_tag_value]) + "/m" else: self.instance_cost = instances_cost_dict[ fqdn_tag_value] instance_cost_column_dict[vpc]['subnets'][subnet][ 'instance_attributes'][instance][ 'instance_cost'] = "$" + str( instances_cost_dict[fqdn_tag_value]) + "/m" elif name_tag_value in instances_cost_dict: self.instance_cost = instances_cost_dict[name_tag_value] if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none'][ 'instance_attributes'][instance][ 'instance_cost'] = "$" + str( instances_cost_dict[name_tag_value]) + "/m" else: self.instance_cost = instances_cost_dict[ fqdn_tag_value] instance_cost_column_dict[vpc]['subnets'][subnet][ 'instance_attributes'][instance][ 'instance_cost'] = "$" + str( instances_cost_dict[name_tag_value]) + "/m" else: if environment == "uncategorized": instance_cost_column_dict[region]['subnets']['none'][ 'instance_attributes'][instance][ 'instance_cost'] = "(empty)" else: self.instance_cost = 0.0 instance_cost_column_dict[vpc]['subnets'][subnet][ 'instance_attributes'][instance][ 'instance_cost'] = "(empty)" if cost_type == 'stack_costs': attr_cost = 0.0 for attribute in stack_attr_list: if attribute == 'instance_cost': attr_cost = self.instance_cost elif attribute == 'storage_cost': attr_cost = self.storage_cost elif attribute == 'total_cost': attr_cost = self.instance_cost + self.storage_cost if stack_attr_details[attribute]['stack'] == ['all']: for apps in apps_in_environment: if stack == apps: if not stack_cost_dict[region][vpc][ subnet][stack][attribute]: stack_cost_dict[region][vpc][subnet][ stack][attribute] = attr_cost else: cost = stack_cost_dict[region][vpc][ subnet][stack][attribute] stack_cost_dict[region][vpc][subnet][ stack][attribute] += attr_cost else: for attr_stack in stack_attr_details[attribute][ 'stack']: if attr_stack == stack: if not stack_cost_dict[region][vpc][ subnet][stack][attribute]: stack_cost_dict[region][vpc][subnet][ stack][attribute] = attr_cost else: stack_cost_dict[region][vpc][subnet][ stack][attribute] += attr_cost stack_cost_string_dict[region][vpc][subnet][stack][attribute] = \ "$"+str(stack_cost_dict[region][vpc][subnet][stack][attribute])+"/m" if cost_type == 'stack_costs': return self.ah_obj.defaultdict_to_dict(stack_cost_string_dict) if cost_type == 'instances_cost': return self.ah_obj.defaultdict_to_dict(instance_cost_column_dict)
class GraphiteHelper(): def __init__(self, request=None, environment=None): self.module = 'graphite_module' self.ah_obj = AtlasHelper() self.aws_helperobj = AwsHelper() self.module_config_data = self.ah_obj.get_atlas_configuration_data( self.module) self.graphite_url = " " self.framework = "" self.parameters_list = [] self.time_interval = 0.0 self.server_monitored = [] self.format = "" self.from_time = "" self.to_time = "" self.memcache_var = memcache.Client([ self.ah_obj.get_atlas_config_data("global_config_data", 'memcache_server_location') ], debug=0) if environment is not None: self.aws_moduleobj = AwsModule(request=request, environment=environment) def get_subnet_list(self, environment): """ Get the subnets for environment which has instances and decide if an attribute should be displayed on a subnet. """ if environment != 'uncategorized': subnets_with_instances = self.aws_moduleobj.get_information( environment, subnets_with_instances='true') subnet_list = [] for subnet, stack_list in subnets_with_instances.iteritems(): for attribute, attr_details in self.module_config_data[ 'stack_attributes'].iteritems(): if attr_details['stack'] == 'all' or set( attr_details['stack']).issubset(set(stack_list)): if subnet not in subnet_list: subnet_list.append(subnet) return subnet_list def get_query_parameters(self): """Get the query parameters from atlas config yaml""" self.graphite_url = self.module_config_data['others'][ 'graphite_url'] + "render/?" self.framework = self.module_config_data['others']['framework'] self.servers_monitored = self.module_config_data['others'][ 'server_name'] self.database = self.module_config_data['others']['database'] self.time_interval = self.module_config_data['others']['time_duration'] if 'from' in self.time_interval: self.from_time = self.time_interval['from'] if 'to' in self.time_interval: self.to_time = self.time_interval['to'] if self.to_time is not None and self.from_time is not None: self.time_string = "&from=" + str(self.from_time) + "&to=" + str( self.to_time) if self.from_time is None: self.time_string = "&to=" + str(self.to_time) if self.to_time is None: self.time_string = "&from=" + str(self.from_time) self.parameters_list = self.module_config_data['others']['parameters'] self.format = self.module_config_data['others']['format'] def queries_for_graphite(self, subnet_list): """Construct queries for grahite""" query_dict = collections.defaultdict(dict) self.get_query_parameters() for subnet in subnet_list: for server in self.servers_monitored: for parameter in self.parameters_list: target = self.framework + "." + subnet + ".ms." + server + "." + self.database + "." + parameter query_dict[subnet][ parameter] = self.graphite_url + "target=" + target + self.time_string + "&format=" + self.format return dict(query_dict) def generate_report(self, query): """Retrieve query results from the graphite server.""" try: report_json = {} response = requests.get(query) if response.status_code == 200: report_json = json.loads( response.text) #convert the json into a python dictionary return report_json except ConnectionError as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) except HTTPError as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_stack_attributes(self, environment): """Get all stack attributes.""" stack_attribute_list, stack_attribute_dict = [], {} for attribute, details in self.module_config_data[ 'stack_attributes'].iteritems(): stack_attribute_list.append( (details['display_name'], details['editable'])) stack_attribute_dict[details['display_name']] = details return (stack_attribute_list, stack_attribute_dict) def get_stack_attribute_values(self, environment): """Get stack attribute values from cache. If it does not exists get it from the the global cache.""" stack_attribute_values = self.memcache_var.get( str(environment + "graphite_stack_attributes")) if not stack_attribute_values: stack_attributes_values = self.memcache_var.get( str(environment + "global_graphite_stack_attributes")) if stack_attribute_values is not None: self.memcache_var.set( str(environment + "graphite_stack_attributes"), stack_attribute_values, 10 * 60) with threading.Lock(): thread = threading.Thread( target=self.cache_stack_attribute_values, args=[environment]) thread.start() return stack_attribute_values def cache_stack_attribute_values(self, environment): """Cache stack attribute values.""" try: stack_attribute_values = self.stack_attribute_values(environment) self.memcache_var.set( str(environment + "graphite_stack_attributes"), stack_attribute_values, 10 * 60) if stack_attribute_values is None: raise Exception( "The graphite attribute values for environment " + environment + " has not been fetched. Please make sure the cache is populated !!!" ) if stack_attribute_values is not None: self.memcache_var.set( str(environment + "global_graphite_stack_attributes"), stack_attribute_values, 15 * 60) self.memcache_var.disconnect_all() except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "cache_stack_attribute_values()", exp_object, exc_type, exc_obj, exc_tb) return {} def stack_attribute_values(self, environment): """get stack attribute values from graphite server and parse it.""" if environment != 'uncategorized': stack_attribute_dict = self.ah_obj.create_nested_defaultdict() organization_list = self.aws_helperobj.get_organizations() region_list = self.aws_helperobj.get_regions() stack_attributes_from_config = self.module_config_data[ 'stack_attributes'] attributes_list = stack_attributes_from_config.keys() subnet_list = self.get_subnet_list(environment) graphite_query_dict = self.queries_for_graphite(subnet_list) for organization in organization_list: for region in region_list: vpc_list = self.aws_helperobj.get_vpc_in_region(region) if vpc_list: for vpc in vpc_list: for subnet in subnet_list: for attribute in stack_attributes_from_config: stack_list = stack_attributes_from_config[ attribute]['stack'] attribute_value = "" suffix = "" if 'suffix' in stack_attributes_from_config[ attribute]: suffix = stack_attributes_from_config[ attribute]['suffix'] display_name = "" if 'display_name' in stack_attributes_from_config[ attribute]: display_name = stack_attributes_from_config[ attribute]['display_name'] report = self.generate_report( graphite_query_dict[subnet] [attribute]) if report: target = self.ah_obj.split_string( report[0]['target'], ('.')) if subnet in target and attribute in target: for index in range( len(report[0] ['datapoints']) - 1, 0, -1): if report and report[0][ 'datapoints'][index][ 0] is not None: attribute_value = str( int(report[0] ['datapoints'] [index][0]) ) + " " + suffix break else: attribute_value = "null" else: attribute_value = "null" for stack in stack_list: stack_attribute_dict[region][vpc][ subnet][stack][ display_name] = attribute_value return self.ah_obj.defaultdict_to_dict(stack_attribute_dict)
class GraphiteHelper(): def __init__(self, request=None, environment=None): self.module = 'graphite_module' self.ah_obj = AtlasHelper() self.aws_helperobj = AwsHelper() self.module_config_data = self.ah_obj.get_atlas_configuration_data(self.module) self.graphite_url = " " self.framework = "" self.parameters_list = [] self.time_interval = 0.0 self.server_monitored = [] self.format = "" self.from_time = "" self.to_time = "" self.memcache_var = memcache.Client([self.ah_obj.get_atlas_config_data("global_config_data",'memcache_server_location')], debug=0) if environment is not None: self.aws_moduleobj = AwsModule(request=request,environment=environment) def get_subnet_list(self, environment): """ Get the subnets for environment which has instances and decide if an attribute should be displayed on a subnet. """ if environment != 'uncategorized': subnets_with_instances = self.aws_moduleobj.get_information(environment, subnets_with_instances='true') subnet_list = [] for subnet, stack_list in subnets_with_instances.iteritems(): for attribute, attr_details in self.module_config_data['stack_attributes'].iteritems(): if attr_details['stack'] == 'all' or set(attr_details['stack']).issubset(set(stack_list)): if subnet not in subnet_list: subnet_list.append(subnet) return subnet_list def get_query_parameters(self): """Get the query parameters from atlas config yaml""" self.graphite_url = self.module_config_data['others']['graphite_url']+"render/?" self.framework = self.module_config_data['others']['framework'] self.servers_monitored = self.module_config_data['others']['server_name'] self.database = self.module_config_data['others']['database'] self.time_interval = self.module_config_data['others']['time_duration'] if 'from' in self.time_interval: self.from_time = self.time_interval['from'] if 'to' in self.time_interval: self.to_time = self.time_interval['to'] if self.to_time is not None and self.from_time is not None: self.time_string = "&from="+str(self.from_time)+"&to="+str(self.to_time) if self.from_time is None: self.time_string = "&to="+str(self.to_time) if self.to_time is None: self.time_string = "&from="+str(self.from_time) self.parameters_list = self.module_config_data['others']['parameters'] self.format = self.module_config_data['others']['format'] def queries_for_graphite(self, subnet_list): """Construct queries for grahite""" query_dict = collections.defaultdict(dict) self.get_query_parameters() for subnet in subnet_list: for server in self.servers_monitored: for parameter in self.parameters_list: target = self.framework+"."+subnet+".ms."+server+"."+self.database+"."+parameter query_dict[subnet][parameter] = self.graphite_url+"target="+target+self.time_string+"&format="+self.format return dict(query_dict) def generate_report(self, query): """Retrieve query results from the graphite server.""" try: report_json = {} response = requests.get(query) if response.status_code == 200: report_json = json.loads(response.text) #convert the json into a python dictionary return report_json except ConnectionError as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) except HTTPError as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "generate_report()", exp_object, exc_type, exc_obj, exc_tb) return {} def get_stack_attributes(self, environment): """Get all stack attributes.""" stack_attribute_list, stack_attribute_dict = [], {} for attribute, details in self.module_config_data['stack_attributes'].iteritems(): stack_attribute_list.append((details['display_name'], details['editable'])) stack_attribute_dict[details['display_name']] = details return(stack_attribute_list, stack_attribute_dict) def get_stack_attribute_values(self, environment): """Get stack attribute values from cache. If it does not exists get it from the the global cache.""" stack_attribute_values = self.memcache_var.get(str(environment+"graphite_stack_attributes")) if not stack_attribute_values: stack_attributes_values = self.memcache_var.get(str(environment+"global_graphite_stack_attributes")) if stack_attribute_values is not None: self.memcache_var.set(str(environment+"graphite_stack_attributes"), stack_attribute_values, 10*60) with threading.Lock(): thread = threading.Thread(target=self.cache_stack_attribute_values, args=[environment]) thread.start() return stack_attribute_values def cache_stack_attribute_values(self, environment): """Cache stack attribute values.""" try: stack_attribute_values = self.stack_attribute_values(environment) self.memcache_var.set(str(environment+"graphite_stack_attributes"), stack_attribute_values, 10*60) if stack_attribute_values is None: raise Exception("The graphite attribute values for environment "+environment+" has not been fetched. Please make sure the cache is populated !!!") if stack_attribute_values is not None: self.memcache_var.set(str(environment+"global_graphite_stack_attributes"),stack_attribute_values, 15*60) self.memcache_var.disconnect_all() except Exception as exp_object: exc_type, exc_obj, exc_tb = sys.exc_info() self.ah_obj.print_exception("graphite_helper.py", "cache_stack_attribute_values()", exp_object, exc_type, exc_obj, exc_tb) return {} def stack_attribute_values(self, environment): """get stack attribute values from graphite server and parse it.""" if environment != 'uncategorized': stack_attribute_dict = self.ah_obj.create_nested_defaultdict() organization_list = self.aws_helperobj.get_organizations() region_list = self.aws_helperobj.get_regions() stack_attributes_from_config = self.module_config_data['stack_attributes'] attributes_list = stack_attributes_from_config.keys() subnet_list = self.get_subnet_list(environment) graphite_query_dict = self.queries_for_graphite(subnet_list) for organization in organization_list: for region in region_list: vpc_list = self.aws_helperobj.get_vpc_in_region(region) if vpc_list: for vpc in vpc_list: for subnet in subnet_list: for attribute in stack_attributes_from_config: stack_list = stack_attributes_from_config[attribute]['stack'] attribute_value="" suffix="" if 'suffix' in stack_attributes_from_config[attribute]: suffix = stack_attributes_from_config[attribute]['suffix'] display_name= "" if 'display_name' in stack_attributes_from_config[attribute]: display_name = stack_attributes_from_config[attribute]['display_name'] report = self.generate_report(graphite_query_dict[subnet][attribute]) if report: target = self.ah_obj.split_string(report[0]['target'], ('.')) if subnet in target and attribute in target: for index in range(len(report[0]['datapoints'])-1, 0, -1): if report and report[0]['datapoints'][index][0] is not None: attribute_value = str(int(report[0]['datapoints'][index][0]))+" "+suffix break else: attribute_value = "null" else:attribute_value = "null" for stack in stack_list: stack_attribute_dict[region][vpc][subnet][stack][display_name] = attribute_value return self.ah_obj.defaultdict_to_dict(stack_attribute_dict)
class AwsInstanceHelper: #class to fetch data from aws for atlas def __init__(self, environment=None): """Initialize variables.""" self.ah_obj = AtlasHelper() self.awshelper_obj = AwsHelper() self.module = 'aws_module' self.environment_dict = {} self.env_subnet_dict = {} self.env_details_dict = {} self.environment_groups = self.ah_obj.get_atlas_config_data("global_config_data", "environment_groups") def get_aws_details(self,regionvpcselected): organizations = self.awshelper_obj.get_organizations() aws_details_dict={} environment_list=[] aws_details = self.awshelper_obj.get_instances_details() for organization in organizations: environment_list = self.awshelper_obj.get_environments(organization) environment_list.append('uncategorized') environment_list.append('all') ah_obj = AtlasHelper() total_count, total_running, total_stopped=0, 0, 0 region_dict = {} region_env_list=[] for environment in environment_list: if environment != 'all': env_attributes = self.ah_obj.get_nested_attribute_values(aws_details, environment)[1] for region in regionvpcselected.keys(): if 'regions' in env_attributes.keys() and region in env_attributes['regions'].keys(): region_keys, region_attributes = ah_obj.get_nested_attribute_values(env_attributes['regions'], region) if 'count' in region_keys and 'running' in region_keys and 'stopped' in region_keys: region_count =region_attributes['count'] region_running =region_attributes['running'] region_stopped =region_attributes['stopped'] if environment == "uncategorized": total_count+=region_count total_running+=region_running total_stopped+=region_stopped region_env_list.append([region, "none", environment, region_count, region_running, region_stopped]) else: vpc_list = regionvpcselected[region] for vpc in vpc_list: if 'vpc' in region_attributes.keys(): vpc_attributes = self.ah_obj.get_nested_attribute_values(region_attributes['vpc'], vpc)[1] vpc_count = vpc_attributes['count'] vpc_running = vpc_attributes['running'] vpc_stopped = vpc_attributes['stopped'] total_count+=vpc_count total_running+=vpc_running total_stopped+=vpc_stopped region_env_list.append([region, vpc, environment, vpc_count, vpc_running, vpc_stopped]) else: region_env_list.append([region, vpc, environment, total_count, total_running, total_stopped]) temp_list, total_list = [], [] for count in range(0, len(environment_list)): total_list.append([environment_list[count],0,0,0]) for index in range(0, len(region_env_list)): for index1 in range(0, len(total_list)): if region_env_list[index][2] == total_list[index1][0]: temp_list = [region_env_list[index][3], region_env_list[index][4], region_env_list[index][5]] total_list[index1][1]+=temp_list[0] total_list[index1][2]+=temp_list[1] total_list[index1][3]+=temp_list[2] aws_details_dict['total_count'] = total_count aws_details_dict['total_running'] = total_running aws_details_dict['total_stopped'] = total_stopped aws_details_dict['environment_list'] = environment_list aws_details_dict['region_dict'] = region_dict aws_details_dict['total_list'] = total_list return aws_details_dict def calculate_environment_aggregates(self, environment, env_attributes, region_vpc_dict): """ Calculate aggregate values for each environment. """ total_list = [0,0,0] for region in region_vpc_dict.keys(): if region in env_attributes['regions'].keys(): region_attributes = env_attributes['regions'][region] if environment == 'uncategorized': region_aggs = [region_attributes['count'],region_attributes['running'], region_attributes['stopped']] total_list = map(add, total_list, region_aggs) else: for vpc in region_vpc_dict[region]: if vpc and vpc in region_attributes['vpc'].keys(): vpc_properties = self.ah_obj.get_nested_attribute_values(env_attributes['regions'][region], vpc)[1] vpc_aggregates = [vpc_properties['count'], vpc_properties['running'], vpc_properties['stopped']] total_list = map(add, total_list, vpc_aggregates) return total_list def get_environment_aggregates(self, environment, region_vpc_dict): """ Get aggregate status values. """ aggregate_list = [0,0,0] organizations = self.awshelper_obj.get_organizations() org_attributes = self.ah_obj.get_nested_attribute_values(self.awshelper_obj.get_instances_details(), 'organizations')[1] for organization in organizations: if self.environment_groups and environment in self.environment_groups[1].keys(): if environment == 'all': env_group_for_environment = self.awshelper_obj.get_environments(organization) else: env_group_for_environment = self.environment_groups[1][environment] for env in env_group_for_environment: env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], env)[1] env_agg_list = self.calculate_environment_aggregates(env, env_attributes, region_vpc_dict) aggregate_list = map(add, aggregate_list, env_agg_list) else: env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], environment)[1] aggregate_list = self.calculate_environment_aggregates(environment, env_attributes, region_vpc_dict) return aggregate_list def determine_environment_status(self, environment, env_attributes, region_vpc_dict): vpc_status_dict = {} for region in region_vpc_dict.keys(): if region not in env_attributes['regions'].keys(): continue else: region_attributes = env_attributes['regions'][region] vpc_status_dict[region] = {'status': []} if environment == 'uncategorized': vpc_status_dict[region]['status'] = [region_attributes['count'], region_attributes['running'], region_attributes['stopped']] else: for vpc in region_vpc_dict[region]: if vpc and vpc in region_attributes['vpc'].keys(): vpc_status_dict[vpc] = {'status': []} vpc_properties = self.ah_obj.get_nested_attribute_values(env_attributes['regions'][region], vpc)[1] vpc_status_dict[vpc]['status'] = [vpc_properties['count'], vpc_properties['running'], vpc_properties['stopped']] return vpc_status_dict def get_environment_status(self, environment, region_vpc_dict): """Get environment status.""" ah_obj = AtlasHelper() vpc_status_dict = {} organizations = self.awshelper_obj.get_organizations() org_attributes = self.ah_obj.get_nested_attribute_values(self.awshelper_obj.get_instances_details(), 'organizations')[1] for organization in organizations: if self.environment_groups and environment in self.environment_groups[1].keys(): env_status_dict = {} if environment == 'all': env_group_for_env = self.awshelper_obj.get_environments(organization) else: env_group_for_env = self.environment_groups[1][environment] for env in env_group_for_env: env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], env)[1] env_status_dict = self.determine_environment_status(env, env_attributes, region_vpc_dict) if not vpc_status_dict: vpc_status_dict.update(env_status_dict) else: for vpc in env_status_dict.keys(): vpc_status_dict[vpc]['status'] = map(add,vpc_status_dict[vpc]['status'],env_status_dict[vpc]['status']) else: env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], environment)[1] vpc_status_dict = self.determine_environment_status(environment, env_attributes, region_vpc_dict) return vpc_status_dict def find_env_details(self, environment, env_attributes, region_vpc_dict): organizations = self.awshelper_obj.get_organizations() details_dict, env_subnets_dict = {}, {} environment_subnets = {'vpc':{}} apps_in_environment, instances_list, application_subnets, env_subnet_list = [], [], [], [] vpc_attribute_dict, aws_info_dict, env_info_dict={}, {}, {} instances_list,apps_in_environment, env_subnet_list = [], [], [] subnets_with_instances = {} #new addition instance_keys = self.ah_obj.get_atlas_config_data(self.module, "instance_keys") for region in region_vpc_dict.keys(): if region not in env_attributes['regions'].keys(): continue else: region_attributes = env_attributes['regions'][region] if environment == 'uncategorized': vpc_attribute_dict[region] = {"subnets":{"none":{"instance_attributes":{}}}} for instance, details in region_attributes['uncat_instances'].iteritems(): if details.has_key('instance_attributes') and details['instance_attributes'].has_key('instance_id'): instance_id = details['instance_attributes']['instance_id'] vpc_attribute_dict[region]["subnets"]["none"]["instance_attributes"][instance_id] = details['instance_attributes'] aws_info_dict[instance_id] = {} aws_info_dict[instance_id]['aws_information'] = details['aws_information'] aws_info_dict[instance_id]['aws_tags'] = details['aws_tags'] else: for vpc in region_vpc_dict[region]: if vpc and vpc in region_attributes['vpc'].keys(): environment_subnets['vpc'][vpc] = {'subnets':[]} env_subnet_list.extend(self.awshelper_obj.get_subnets_in_environment(region, vpc, environment)) environment_subnets['vpc'][vpc]['subnets'] = env_subnet_list vpc_attribute_dict[vpc] = {'subnets': {}, 'applications':[]} vpc_attribute_dict[vpc]['subnets'] = {} vpc_properties = self.ah_obj.get_nested_attribute_values(env_attributes['regions'][region], vpc)[1] if 'subnets' in vpc_properties.keys(): for subnet in vpc_properties['subnets'].keys(): instances_details = vpc_properties['subnets'][subnet]['instances'] if instances_details: #env_subnet_list.append(subnet) subnets_with_instances[subnet] = [] vpc_attribute_dict[vpc]['subnets'][subnet] = {'instance_attributes': {}} if subnet not in application_subnets: application_subnets.append(subnet) for instance, details in instances_details.iteritems(): instance_attributes = details['instance_attributes'] if instance_attributes: details_key = details['instance_attributes']['instance_id'] vpc_attribute_dict[vpc]['subnets'][subnet]['instance_attributes'][details_key] = instance_attributes aws_info_dict[details_key] = {} aws_info_dict[details_key]['aws_information'] = details['aws_information'] aws_info_dict[details_key]['aws_tags'] = details['aws_tags'] instance_application = instance_attributes['application'] if instance_application not in subnets_with_instances[subnet]: subnets_with_instances[subnet].append(instance_application) if instance_attributes['application'] not in apps_in_environment: apps_in_environment.append(instance_attributes['application']) details_dict['subnets_with_instances'] = subnets_with_instances details_dict['environment_subnets'] = environment_subnets details_dict['application_subnets'] = application_subnets details_dict['vpc_attribute_dict'] = vpc_attribute_dict details_dict['aws_info_dict'] = aws_info_dict details_dict['env_subnet_list'] = env_subnet_list details_dict['apps_in_environment'] = apps_in_environment return details_dict def get_environment_details(self, environment, region_vpc_dict): """ Get details for a particular environment or environment_group. """ organizations = self.awshelper_obj.get_organizations() org_attributes = self.ah_obj.get_nested_attribute_values(self.awshelper_obj.get_instances_details(), 'organizations')[1] for organization in organizations: if self.environment_groups and environment in self.environment_groups[0]: if environment == 'all': env_group_for_environment = self.awshelper_obj.get_environments(organization) else: env_group_for_environment = self.environment_groups[1][environment] for env in env_group_for_environment: env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], env)[1] environment_details = self.find_env_details(env, env_attributes, region_vpc_dict) if not self.env_details_dict: self.env_details_dict.update(environment_details) else: generator_obj = self.ah_obj.merge_dictionaries(self.env_details_dict, environment_details) self.env_details_dict = {key:value for key, value in generator_obj} else: env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], environment)[1] self.env_details_dict = self.find_env_details(environment, env_attributes, region_vpc_dict) return self.env_details_dict def get_column_data(self, environment): column_dict = {} if 'vpc_attribute_dict' in self.env_details_dict: environment_attributes = self.env_details_dict['vpc_attribute_dict'] for vpc, vpc_attrs_dict in environment_attributes.iteritems(): column_dict[vpc] = {'subnets': {}} if 'subnets' in vpc_attrs_dict: for subnet , subnet_attributes in vpc_attrs_dict['subnets'].iteritems(): column_dict[vpc]['subnets'][subnet] = {'instance_attributes':[]} if 'instance_attributes' in subnet_attributes: column_dict[vpc]['subnets'][subnet]['instance_attributes'] = subnet_attributes['instance_attributes'] return column_dict
class JenkinsModule(AtlasBase): def __init__(self, request=None, environment=None): self.ah_obj = AtlasHelper() self.aws_helperobj = AwsHelper() self.module = "jenkins_module" if environment is None: self.awsact_obj = AwsActions() self.jenkinsact_obj = JenkinsActions(request) else: self.jenkinsact_obj = JenkinsActions(request,environment) def get_configuration_data(self, key): value = self.ah_obj.get_atlas_config_data(self.module, key) if isinstance(value, dict): return value[0] else: return value def get_information(self, environment, **kwargs): pass def get_status(self, environment=None): pass def get_tabs(self, environment=None): pass def get_instance_actions(self, environment=None): pass def get_instance_group_actions(self): pass def get_stack_actions(self, environment=None): stack_actions = self.get_configuration_data('stack_actions') if stack_actions: return stack_actions[0] def get_vpc_actions(self): vpc_actions = self.get_configuration_data("vpc_actions") if vpc_actions: return vpc_actions[0] def get_parameter_values(self, environment, parameter): return self.jenkinsact_obj.parameters_values(environment, parameter) def get_action_parameters(self, action_type, environment=None): return self.jenkinsact_obj.action_parameters(action_type, environment) def perform_instance_actions(): pass def perform_instancegroup_actions(): pass def perform_stack_actions(self, json_data, environment=None): action = json_data['action'] parameters_dict = json_data['parameters'] initial_status = self.jenkinsact_obj.initiate_actions(action, parameters_dict) if initial_status is not None: initial_status['action_type'] = json_data['action_type'] initial_status['start_time'] = json_data['start_time'] initial_status['action'] = json_data['action'] return initial_status def perform_vpc_actions(self, json_data, environment=None): action = json_data['action'] action_type = json_data['action_type'] parameters_dict = json_data['parameters'] initial_status = self.jenkinsact_obj.initiate_actions(action, parameters_dict) if initial_status is not None: initial_status['action_type'] = json_data['action_type'] initial_status['start_time'] = json_data['start_time'] initial_status['action'] = json_data['action'] return initial_status def get_action_status(self, json_data, environment=None): action = json_data['action'] action_type = json_data['action_type'] parameter_dict = json_data['parameters'] action_status = self.jenkinsact_obj.action_state(action) if action_status is not None: action_status['action_type'] = json_data['action_type'] action_status['start_time'] = json_data['start_time'] action_status['action'] = json_data['action'] return action_status def get_columns(self, environment=None): pass def load_session(self, request, environment=None): pass def save_session(self, request, environment=None): pass def get_defaults(): pass def get_aggregates(self, environment=None): pass def get_stack_attributes(self, environment=None, stack=None): stack_attribute_list = [] stack_attributes_dict = self.ah_obj.get_atlas_config_data('jenkins_module', 'stack_attributes')[1] for attribute, details in stack_attributes_dict.iteritems(): stack_attribute_list.append((attribute, details['editable'])) return(stack_attribute_list, stack_attributes_dict) def get_attribute_values(self, environment=None): jenkins_build_infodict = self.jenkinsact_obj.get_jenkins_build_userinfo('AWS-Build-Dev-Deploy-Dev') rev_sorted_buildno_list = list(reversed(sorted(jenkins_build_infodict.keys()))) (stack_attr_list, stack_attr_details) = self.get_configuration_data("stack_attributes") stack_attr_values_dict = self.ah_obj.create_nested_defaultdict() organization_list = self.aws_helperobj.get_organizations() temp_subnet_list = [] for organization in organization_list: region_list = self.aws_helperobj.get_regions() for region in region_list: vpc_list = self.aws_helperobj.get_vpc_in_region(region) if vpc_list: for vpc in vpc_list: for attribute, attr_details in stack_attr_details.iteritems(): for build_number in rev_sorted_buildno_list: subnet = jenkins_build_infodict[build_number]['subnet'] if subnet not in temp_subnet_list: temp_subnet_list.append(subnet) for attribute in stack_attr_list: for stack in stack_attr_details[attribute]['stack']: if attribute in jenkins_build_infodict[build_number]: stack_attr_values_dict[region][vpc][subnet][stack][attribute] = \ jenkins_build_infodict[build_number][attribute] return self.ah_obj.defaultdict_to_dict(stack_attr_values_dict) def refresh_information(self, environment=None): self.jenkinsact_obj.cache_jenkins_build_userinfo()