def get_aws_details(self, regionvpcselected):
organizations = self.awshelper_obj.get_organizations()
aws_details_dict = {}
environment_list = []
aws_details = self.awshelper_obj.get_instances_details()
for organization in organizations:
environment_list = self.awshelper_obj.get_environments(
organization)
environment_list.append('uncategorized')
environment_list.append('all')
ah_obj = AtlasHelper()
total_count, total_running, total_stopped = 0, 0, 0
region_dict = {}
region_env_list = []
for environment in environment_list:
if environment != 'all':
env_attributes = self.ah_obj.get_nested_attribute_values(
aws_details, environment)[1]
for region in regionvpcselected.keys():
if 'regions' in env_attributes.keys(
) and region in env_attributes['regions'].keys():
region_keys, region_attributes = ah_obj.get_nested_attribute_values(
env_attributes['regions'], region)
if 'count' in region_keys and 'running' in region_keys and 'stopped' in region_keys:
region_count = region_attributes['count']
region_running = region_attributes['running']
region_stopped = region_attributes['stopped']
if environment == "uncategorized":
total_count += region_count
total_running += region_running
total_stopped += region_stopped
region_env_list.append([
region, "none", environment,
region_count, region_running,
region_stopped
])
else:
vpc_list = regionvpcselected[region]
for vpc in vpc_list:
if 'vpc' in region_attributes.keys():
vpc_attributes = self.ah_obj.get_nested_attribute_values(
region_attributes['vpc'],
vpc)[1]
vpc_count = vpc_attributes['count']
vpc_running = vpc_attributes[
'running']
vpc_stopped = vpc_attributes[
'stopped']
total_count += vpc_count
total_running += vpc_running
total_stopped += vpc_stopped
region_env_list.append([
region, vpc, environment, vpc_count,
vpc_running, vpc_stopped
])
else:
region_env_list.append([
region, vpc, environment, total_count, total_running,
total_stopped
])
temp_list, total_list = [], []
for count in range(0, len(environment_list)):
total_list.append([environment_list[count], 0, 0, 0])
for index in range(0, len(region_env_list)):
for index1 in range(0, len(total_list)):
if region_env_list[index][2] == total_list[index1][0]:
temp_list = [
region_env_list[index][3], region_env_list[index][4],
region_env_list[index][5]
]
total_list[index1][1] += temp_list[0]
total_list[index1][2] += temp_list[1]
total_list[index1][3] += temp_list[2]
aws_details_dict['total_count'] = total_count
aws_details_dict['total_running'] = total_running
aws_details_dict['total_stopped'] = total_stopped
aws_details_dict['environment_list'] = environment_list
aws_details_dict['region_dict'] = region_dict
aws_details_dict['total_list'] = total_list
return aws_details_dict
class CloudabilityModule(AtlasBase):
def __init__(self, request=None,environment=None):
self.module = "cloudability_module"
self.ah_obj = AtlasHelper()
self.cloud_obj = Cloudability()
self.awshelper_obj = AwsHelper()
self.request = request
self.aws_object = AwsModule(request, environment)
self.instance_cost = 0.0
self.storage_cost = 0.0
def get_information(self, environment=None, **kwargs):
organization_list = self.awshelper_obj.get_organizations()
if environment is None:
if 'env_cost_dict' in kwargs:
if kwargs['env_cost_dict'] == 'true':
for organization in organization_list:
env_cost_dict= self.cloud_obj.get_cloudability_costs()['environment_costs'][organization]
env_cost_dict['all'] = self.get_information(ec2_cost_dict='true')['region_zone']
return env_cost_dict
if 'ec2_cost_dict' in kwargs:
if kwargs['ec2_cost_dict'] == 'true':
for organization in organization_list:
ec2_costs= self.cloud_obj.get_cloudability_costs()['ec2_costs'][organization]
return ec2_costs
else:
if 'env_cost_dict' in kwargs:
if kwargs['env_cost_dict'] == 'true':
env_costs = 0
for organization in organization_list:
env_cost_dict = self.cloud_obj.get_cloudability_costs()['environment_costs'][organization]
environment_groups = self.ah_obj.get_atlas_config_data("global_config_data", "environment_groups")
if environment_groups and environment in environment_groups[1].keys():
if environment == 'all':
env_group_for_environment = self.awshelper_obj.get_environments(organization)
else:
env_group_for_environment = environment_groups[1][environment]
for env in env_group_for_environment:
env_costs+=env_cost_dict[env]
env_cost_dict[environment] = env_costs
return env_cost_dict
if 'apps_in_environment' in kwargs:
if kwargs['apps_in_environment'] == 'true':
return self.aws_object.get_information(environment, apps_in_environment='true')
if 'instance_data' in kwargs:
if kwargs['instance_data'] == 'true':
return self.aws_object.get_information(environment, instance_data='true')
if 'instances_cost_dict' in kwargs:
if kwargs['instances_cost_dict'] == 'true':
for organization in organization_list:
return self.cloud_obj.get_cloudability_costs()['instances_costs'][organization]
if 'ebs_cost_dict' in kwargs:
if kwargs['ebs_cost_dict'] == 'true':
for organization in organization_list:
return self.cloud_obj.get_cloudability_costs()['ebs_costs'][organization]
if 'aws_info_dict' in kwargs:
if kwargs['aws_info_dict'] == 'true':
return self.aws_object.get_information(environment, 'aws_info_dict')
if 'application_subnets' in kwargs:
if kwargs['application_subnets'] == 'true':
return self.aws_object.get_information(environment, 'application_subnets')
def get_configuration_data(self, key):
value = self.ah_obj.get_atlas_config_data(self.module, key)
if isinstance(value, dict):
return value[0]
else:
return value
def get_stack_attributes(self, environment=None):
"""
Get stack attributes from config file.
"""
stack_attribute_list = []
stack_attributes_dict = self.ah_obj.get_atlas_config_data('cloudability_module', 'stack_attributes')[1]
for attribute, details in stack_attributes_dict.iteritems():
stack_attribute_list.append((attribute, details['editable']))
return(stack_attribute_list, stack_attributes_dict)
def get_attribute_values(self, environment=None):
return self.__get_detailed_instances_cost_dict(environment, 'stack_costs')
def get_status(self,environment=None):
status_information = self.get_configuration_data('status')
cloud_status_dict = {}
organization_list = self.awshelper_obj.get_organizations()
environment_list = []
if environment==None:
env_cost_dict = self.get_information(env_cost_dict='true')
cloud_status_dict = {environment: ["$"+str(env_cost_dict[environment])] for environment in env_cost_dict.keys()}
else:
env_cost_dict = self.get_information(environment, env_cost_dict='true')
region_vpc_selection = self.aws_object.get_information(environment, region_vpc_dict='true')
if environment == "uncategorized":
region_list = self.awshelper_obj.get_regions()
for region in region_vpc_selection:
if region =='east':
cloud_status_dict[region] = ["$"+str(env_cost_dict[environment])]
else:
cloud_status_dict[region] = ["$"+ "0.0"]
else:
for vpc in ['ame1']: #should be changed later to include all vpcs.
cloud_status_dict[vpc] = ["$"+str(env_cost_dict[environment])]
return (status_information, cloud_status_dict)
def get_tabs(self, environment=None):
pass
def get_instance_actions(self, environment=None):
pass
def get_environment_actions(self, environment=None):
pass
def get_instance_group_actions(self, environment=None):
pass
def get_stack_actions(self, environment=None):
pass
def get_vpc_actions(self):
pass
def get_action_status(self, json_data, environment=None):
pass
def perform_instance_actions(self, environment=None):
pass
def perform_instancegroup_actions():
pass
def perform_stack_actions():
pass
def perform_vpc_actions(self, json_data):
pass
def perform_environment_actions(self, environment=None):
pass
def get_columns(self, environment=None):
column_list = self.ah_obj.get_atlas_config_data(self.module, 'columns')
column_dict= self.ah_obj.create_nested_defaultdict()
if column_list:
column_dict = self.__get_detailed_instances_cost_dict(environment, 'instances_cost')
return (column_list, self.ah_obj.defaultdict_to_dict(column_dict))
def get_action_parameters(self, action_type, environment=None):
pass
def load_session(self, request, environment=None):
pass
def save_session(self, request, environment=None):
pass
def get_defaults():
pass
def get_aggregates(self, environment=None):
aggregates = self.ah_obj.get_atlas_config_data(self.module, 'aggregates')
if environment is None:
aggregate_list = ["$"+str(self.get_information(ec2_cost_dict='true')['region_zone'])]
return (aggregates, aggregate_list)
else:
aggregate_dict = collections.defaultdict(dict)
for agg_key in aggregates:
if agg_key == 'cost':
aggregate_dict[agg_key] = self.get_information(env_cost_dict='true')[environment]
return dict(aggregate_dict)
def refresh_information(self, environment=None):
self.cloud_obj.cache_cloud_costs()
return
def __get_detailed_instances_cost_dict(self, environment, cost_type):
instances_cost_dict = self.get_information(environment, instances_cost_dict = 'true')
ebs_cost_dict = self.get_information(environment, ebs_cost_dict = 'true')
aws_tabs_dict = self.aws_object.get_tabs(environment)[1]
instance_information = self.aws_object.get_information(environment, instance_data='true')
organization_list = self.awshelper_obj.get_organizations()
(stack_attr_list, stack_attr_details) = self.get_configuration_data('stack_attributes')
apps_in_environment = self.get_information(environment, apps_in_environment='true')
application_subnets = self.get_information(environment, application_subnets='true')
region, vpc, subnet = "", "", ""
instance_cost_column_dict= self.ah_obj.create_nested_defaultdict()
ebs_cost_column_dict = self.ah_obj.create_nested_defaultdict()
stack_cost_dict= self.ah_obj.create_nested_defaultdict()
stack_cost_string_dict = self.ah_obj.create_nested_defaultdict()
name_tag_value, fqdn_tag_value = "", ""
for instance, aws_tabs_dict in aws_tabs_dict.iteritems():
attribute_cost={}
if 'Name' in aws_tabs_dict['aws_tags']: name_tag_value = aws_tabs_dict['aws_tags']["Name"]
if 'fqdn' in aws_tabs_dict['aws_tags']: fqdn_tag_value = aws_tabs_dict['aws_tags']['fqdn']
instance_details = self.ah_obj.get_nested_attribute_values(instance_information, instance)[1]
region = instance_details['region'] if "region" in instance_details else "none"
subnet = instance_details['subnet'] if "subnet" in instance_details else "none"
attribute_cost[subnet] = {}
vpc = instance_details['vpc'] if "vpc" in instance_details else "none"
attribute_cost[vpc] = {}
attribute_cost[vpc][subnet] = {}
stack = instance_details['application'] if "application" in instance_details else "none"
if cost_type == 'instances_cost' or cost_type == 'stack_costs':
if fqdn_tag_value in ebs_cost_dict:
self.storage_cost = ebs_cost_dict[fqdn_tag_value]
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['storage_cost'] = "$" +str(ebs_cost_dict[fqdn_tag_value])+"/m"
else:
instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['storage_cost'] = "$" +str(ebs_cost_dict[fqdn_tag_value])+"/m"
else:
self.storage_cost = 0.0
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['storage_cost'] = "$" + "0.0"+"/m"
else:
instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['storage_cost'] = "$" + "0.0"+"/m"
if fqdn_tag_value in instances_cost_dict:
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[fqdn_tag_value])+"/m"
else:
self.instance_cost = instances_cost_dict[fqdn_tag_value]
instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[fqdn_tag_value])+"/m"
elif name_tag_value in instances_cost_dict:
self.instance_cost = instances_cost_dict[name_tag_value]
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[name_tag_value]) +"/m"
else:
self.instance_cost = instances_cost_dict[fqdn_tag_value]
instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['instance_cost'] = "$" + str(instances_cost_dict[name_tag_value]) +"/m"
else:
if environment=="uncategorized":
instance_cost_column_dict[region]['subnets']['none']['instance_attributes'][instance]['instance_cost'] = "(empty)"
else:
self.instance_cost = 0.0
instance_cost_column_dict[vpc]['subnets'][subnet]['instance_attributes'][instance]['instance_cost'] = "(empty)"
if cost_type == 'stack_costs':
attr_cost = 0.0
for attribute in stack_attr_list:
if attribute == 'instance_cost':
attr_cost = self.instance_cost
elif attribute == 'storage_cost':
attr_cost = self.storage_cost
elif attribute == 'total_cost':
attr_cost = self.instance_cost + self.storage_cost
if stack_attr_details[attribute]['stack'] == ['all']:
for apps in apps_in_environment:
if stack == apps:
if not stack_cost_dict[region][vpc][subnet][stack][attribute]:
stack_cost_dict[region][vpc][subnet][stack][attribute] = attr_cost
else:
cost = stack_cost_dict[region][vpc][subnet][stack][attribute]
stack_cost_dict[region][vpc][subnet][stack][attribute]+=attr_cost
else:
for attr_stack in stack_attr_details[attribute]['stack']:
if attr_stack == stack:
if not stack_cost_dict[region][vpc][subnet][stack][attribute]:
stack_cost_dict[region][vpc][subnet][stack][attribute] = attr_cost
else:
stack_cost_dict[region][vpc][subnet][stack][attribute]+=attr_cost
stack_cost_string_dict[region][vpc][subnet][stack][attribute] = \
"$"+str(stack_cost_dict[region][vpc][subnet][stack][attribute])+"/m"
if cost_type == 'stack_costs': return self.ah_obj.defaultdict_to_dict(stack_cost_string_dict)
if cost_type == 'instances_cost' : return self.ah_obj.defaultdict_to_dict(instance_cost_column_dict)
class AwsInstanceHelper:
#class to fetch data from aws for atlas
def __init__(self, environment=None):
"""Initialize variables."""
self.ah_obj = AtlasHelper()
self.awshelper_obj = AwsHelper()
self.module = 'aws_module'
self.environment_dict = {}
self.env_subnet_dict = {}
self.env_details_dict = {}
self.environment_groups = self.ah_obj.get_atlas_config_data(
"global_config_data", "environment_groups")
def get_aws_details(self, regionvpcselected):
organizations = self.awshelper_obj.get_organizations()
aws_details_dict = {}
environment_list = []
aws_details = self.awshelper_obj.get_instances_details()
for organization in organizations:
environment_list = self.awshelper_obj.get_environments(
organization)
environment_list.append('uncategorized')
environment_list.append('all')
ah_obj = AtlasHelper()
total_count, total_running, total_stopped = 0, 0, 0
region_dict = {}
region_env_list = []
for environment in environment_list:
if environment != 'all':
env_attributes = self.ah_obj.get_nested_attribute_values(
aws_details, environment)[1]
for region in regionvpcselected.keys():
if 'regions' in env_attributes.keys(
) and region in env_attributes['regions'].keys():
region_keys, region_attributes = ah_obj.get_nested_attribute_values(
env_attributes['regions'], region)
if 'count' in region_keys and 'running' in region_keys and 'stopped' in region_keys:
region_count = region_attributes['count']
region_running = region_attributes['running']
region_stopped = region_attributes['stopped']
if environment == "uncategorized":
total_count += region_count
total_running += region_running
total_stopped += region_stopped
region_env_list.append([
region, "none", environment,
region_count, region_running,
region_stopped
])
else:
vpc_list = regionvpcselected[region]
for vpc in vpc_list:
if 'vpc' in region_attributes.keys():
vpc_attributes = self.ah_obj.get_nested_attribute_values(
region_attributes['vpc'],
vpc)[1]
vpc_count = vpc_attributes['count']
vpc_running = vpc_attributes[
'running']
vpc_stopped = vpc_attributes[
'stopped']
total_count += vpc_count
total_running += vpc_running
total_stopped += vpc_stopped
region_env_list.append([
region, vpc, environment, vpc_count,
vpc_running, vpc_stopped
])
else:
region_env_list.append([
region, vpc, environment, total_count, total_running,
total_stopped
])
temp_list, total_list = [], []
for count in range(0, len(environment_list)):
total_list.append([environment_list[count], 0, 0, 0])
for index in range(0, len(region_env_list)):
for index1 in range(0, len(total_list)):
if region_env_list[index][2] == total_list[index1][0]:
temp_list = [
region_env_list[index][3], region_env_list[index][4],
region_env_list[index][5]
]
total_list[index1][1] += temp_list[0]
total_list[index1][2] += temp_list[1]
total_list[index1][3] += temp_list[2]
aws_details_dict['total_count'] = total_count
aws_details_dict['total_running'] = total_running
aws_details_dict['total_stopped'] = total_stopped
aws_details_dict['environment_list'] = environment_list
aws_details_dict['region_dict'] = region_dict
aws_details_dict['total_list'] = total_list
return aws_details_dict
def calculate_environment_aggregates(self, environment, env_attributes,
region_vpc_dict):
"""
Calculate aggregate values for each environment.
"""
total_list = [0, 0, 0]
for region in region_vpc_dict.keys():
if region in env_attributes['regions'].keys():
region_attributes = env_attributes['regions'][region]
if environment == 'uncategorized':
region_aggs = [
region_attributes['count'],
region_attributes['running'],
region_attributes['stopped']
]
total_list = map(add, total_list, region_aggs)
else:
for vpc in region_vpc_dict[region]:
if vpc and vpc in region_attributes['vpc'].keys():
vpc_properties = self.ah_obj.get_nested_attribute_values(
env_attributes['regions'][region], vpc)[1]
vpc_aggregates = [
vpc_properties['count'],
vpc_properties['running'],
vpc_properties['stopped']
]
total_list = map(add, total_list, vpc_aggregates)
return total_list
def get_environment_aggregates(self, environment, region_vpc_dict):
"""
Get aggregate status values.
"""
aggregate_list = [0, 0, 0]
organizations = self.awshelper_obj.get_organizations()
org_attributes = self.ah_obj.get_nested_attribute_values(
self.awshelper_obj.get_instances_details(), 'organizations')[1]
for organization in organizations:
if self.environment_groups and environment in self.environment_groups[
1].keys():
if environment == 'all':
env_group_for_environment = self.awshelper_obj.get_environments(
organization)
else:
env_group_for_environment = self.environment_groups[1][
environment]
for env in env_group_for_environment:
env_attributes = self.ah_obj.get_nested_attribute_values(
org_attributes[organization], env)[1]
env_agg_list = self.calculate_environment_aggregates(
env, env_attributes, region_vpc_dict)
aggregate_list = map(add, aggregate_list, env_agg_list)
else:
env_attributes = self.ah_obj.get_nested_attribute_values(
org_attributes[organization], environment)[1]
aggregate_list = self.calculate_environment_aggregates(
environment, env_attributes, region_vpc_dict)
return aggregate_list
def determine_environment_status(self, environment, env_attributes,
region_vpc_dict):
vpc_status_dict = {}
for region in region_vpc_dict.keys():
if region not in env_attributes['regions'].keys():
continue
else:
region_attributes = env_attributes['regions'][region]
vpc_status_dict[region] = {'status': []}
if environment == 'uncategorized':
vpc_status_dict[region]['status'] = [
region_attributes['count'],
region_attributes['running'],
region_attributes['stopped']
]
else:
for vpc in region_vpc_dict[region]:
if vpc and vpc in region_attributes['vpc'].keys():
vpc_status_dict[vpc] = {'status': []}
vpc_properties = self.ah_obj.get_nested_attribute_values(
env_attributes['regions'][region], vpc)[1]
vpc_status_dict[vpc]['status'] = [
vpc_properties['count'],
vpc_properties['running'],
vpc_properties['stopped']
]
return vpc_status_dict
def get_environment_status(self, environment, region_vpc_dict):
"""Get environment status."""
ah_obj = AtlasHelper()
vpc_status_dict = {}
organizations = self.awshelper_obj.get_organizations()
org_attributes = self.ah_obj.get_nested_attribute_values(
self.awshelper_obj.get_instances_details(), 'organizations')[1]
for organization in organizations:
if self.environment_groups and environment in self.environment_groups[
1].keys():
env_status_dict = {}
if environment == 'all':
env_group_for_env = self.awshelper_obj.get_environments(
organization)
else:
env_group_for_env = self.environment_groups[1][environment]
for env in env_group_for_env:
env_attributes = self.ah_obj.get_nested_attribute_values(
org_attributes[organization], env)[1]
env_status_dict = self.determine_environment_status(
env, env_attributes, region_vpc_dict)
if not vpc_status_dict:
vpc_status_dict.update(env_status_dict)
else:
for vpc in env_status_dict.keys():
vpc_status_dict[vpc]['status'] = map(
add, vpc_status_dict[vpc]['status'],
env_status_dict[vpc]['status'])
else:
env_attributes = self.ah_obj.get_nested_attribute_values(
org_attributes[organization], environment)[1]
vpc_status_dict = self.determine_environment_status(
environment, env_attributes, region_vpc_dict)
return vpc_status_dict
def find_env_details(self, environment, env_attributes, region_vpc_dict):
organizations = self.awshelper_obj.get_organizations()
details_dict, env_subnets_dict = {}, {}
environment_subnets = {'vpc': {}}
apps_in_environment, instances_list, application_subnets, env_subnet_list = [], [], [], []
vpc_attribute_dict, aws_info_dict, env_info_dict = {}, {}, {}
instances_list, apps_in_environment, env_subnet_list = [], [], []
subnets_with_instances = {} #new addition
instance_keys = self.ah_obj.get_atlas_config_data(
self.module, "instance_keys")
for region in region_vpc_dict.keys():
if region not in env_attributes['regions'].keys():
continue
else:
region_attributes = env_attributes['regions'][region]
if environment == 'uncategorized':
vpc_attribute_dict[region] = {
"subnets": {
"none": {
"instance_attributes": {}
}
}
}
for instance, details in region_attributes[
'uncat_instances'].iteritems():
if details.has_key('instance_attributes') and details[
'instance_attributes'].has_key('instance_id'):
instance_id = details['instance_attributes'][
'instance_id']
vpc_attribute_dict[region]["subnets"]["none"][
"instance_attributes"][instance_id] = details[
'instance_attributes']
aws_info_dict[instance_id] = {}
aws_info_dict[instance_id][
'aws_information'] = details['aws_information']
aws_info_dict[instance_id]['aws_tags'] = details[
'aws_tags']
else:
for vpc in region_vpc_dict[region]:
if vpc and vpc in region_attributes['vpc'].keys():
environment_subnets['vpc'][vpc] = {'subnets': []}
env_subnet_list.extend(
self.awshelper_obj.get_subnets_in_environment(
region, vpc, environment))
environment_subnets['vpc'][vpc][
'subnets'] = env_subnet_list
vpc_attribute_dict[vpc] = {
'subnets': {},
'applications': []
}
vpc_attribute_dict[vpc]['subnets'] = {}
vpc_properties = self.ah_obj.get_nested_attribute_values(
env_attributes['regions'][region], vpc)[1]
if 'subnets' in vpc_properties.keys():
for subnet in vpc_properties['subnets'].keys():
instances_details = vpc_properties[
'subnets'][subnet]['instances']
if instances_details:
#env_subnet_list.append(subnet)
subnets_with_instances[subnet] = []
vpc_attribute_dict[vpc]['subnets'][
subnet] = {
'instance_attributes': {}
}
if subnet not in application_subnets:
application_subnets.append(subnet)
for instance, details in instances_details.iteritems(
):
instance_attributes = details[
'instance_attributes']
if instance_attributes:
details_key = details[
'instance_attributes'][
'instance_id']
vpc_attribute_dict[vpc][
'subnets'][subnet][
'instance_attributes'][
details_key] = instance_attributes
aws_info_dict[details_key] = {}
aws_info_dict[details_key][
'aws_information'] = details[
'aws_information']
aws_info_dict[details_key][
'aws_tags'] = details[
'aws_tags']
instance_application = instance_attributes[
'application']
if instance_application not in subnets_with_instances[
subnet]:
subnets_with_instances[
subnet].append(
instance_application)
if instance_attributes[
'application'] not in apps_in_environment:
apps_in_environment.append(
instance_attributes[
'application'])
details_dict['subnets_with_instances'] = subnets_with_instances
details_dict['environment_subnets'] = environment_subnets
details_dict['application_subnets'] = application_subnets
details_dict['vpc_attribute_dict'] = vpc_attribute_dict
details_dict['aws_info_dict'] = aws_info_dict
details_dict['env_subnet_list'] = env_subnet_list
details_dict['apps_in_environment'] = apps_in_environment
return details_dict
def get_environment_details(self, environment, region_vpc_dict):
"""
Get details for a particular environment or environment_group.
"""
organizations = self.awshelper_obj.get_organizations()
org_attributes = self.ah_obj.get_nested_attribute_values(
self.awshelper_obj.get_instances_details(), 'organizations')[1]
for organization in organizations:
if self.environment_groups and environment in self.environment_groups[
0]:
if environment == 'all':
env_group_for_environment = self.awshelper_obj.get_environments(
organization)
else:
env_group_for_environment = self.environment_groups[1][
environment]
for env in env_group_for_environment:
env_attributes = self.ah_obj.get_nested_attribute_values(
org_attributes[organization], env)[1]
environment_details = self.find_env_details(
env, env_attributes, region_vpc_dict)
if not self.env_details_dict:
self.env_details_dict.update(environment_details)
else:
generator_obj = self.ah_obj.merge_dictionaries(
self.env_details_dict, environment_details)
self.env_details_dict = {
key: value
for key, value in generator_obj
}
else:
env_attributes = self.ah_obj.get_nested_attribute_values(
org_attributes[organization], environment)[1]
self.env_details_dict = self.find_env_details(
environment, env_attributes, region_vpc_dict)
return self.env_details_dict
def get_column_data(self, environment):
column_dict = {}
if 'vpc_attribute_dict' in self.env_details_dict:
environment_attributes = self.env_details_dict[
'vpc_attribute_dict']
for vpc, vpc_attrs_dict in environment_attributes.iteritems():
column_dict[vpc] = {'subnets': {}}
if 'subnets' in vpc_attrs_dict:
for subnet, subnet_attributes in vpc_attrs_dict[
'subnets'].iteritems():
column_dict[vpc]['subnets'][subnet] = {
'instance_attributes': []
}
if 'instance_attributes' in subnet_attributes:
column_dict[vpc]['subnets'][subnet][
'instance_attributes'] = subnet_attributes[
'instance_attributes']
return column_dict
class CloudabilityModule(AtlasBase):
def __init__(self, request=None, environment=None):
self.module = "cloudability_module"
self.ah_obj = AtlasHelper()
self.cloud_obj = Cloudability()
self.awshelper_obj = AwsHelper()
self.request = request
self.aws_object = AwsModule(request, environment)
self.instance_cost = 0.0
self.storage_cost = 0.0
def get_information(self, environment=None, **kwargs):
organization_list = self.awshelper_obj.get_organizations()
if environment is None:
if 'env_cost_dict' in kwargs:
if kwargs['env_cost_dict'] == 'true':
for organization in organization_list:
env_cost_dict = self.cloud_obj.get_cloudability_costs(
)['environment_costs'][organization]
env_cost_dict['all'] = self.get_information(
ec2_cost_dict='true')['region_zone']
return env_cost_dict
if 'ec2_cost_dict' in kwargs:
if kwargs['ec2_cost_dict'] == 'true':
for organization in organization_list:
ec2_costs = self.cloud_obj.get_cloudability_costs(
)['ec2_costs'][organization]
return ec2_costs
else:
if 'env_cost_dict' in kwargs:
if kwargs['env_cost_dict'] == 'true':
env_costs = 0
for organization in organization_list:
env_cost_dict = self.cloud_obj.get_cloudability_costs(
)['environment_costs'][organization]
environment_groups = self.ah_obj.get_atlas_config_data(
"global_config_data", "environment_groups")
if environment_groups and environment in environment_groups[
1].keys():
if environment == 'all':
env_group_for_environment = self.awshelper_obj.get_environments(
organization)
else:
env_group_for_environment = environment_groups[
1][environment]
for env in env_group_for_environment:
env_costs += env_cost_dict[env]
env_cost_dict[environment] = env_costs
return env_cost_dict
if 'apps_in_environment' in kwargs:
if kwargs['apps_in_environment'] == 'true':
return self.aws_object.get_information(
environment, apps_in_environment='true')
if 'instance_data' in kwargs:
if kwargs['instance_data'] == 'true':
return self.aws_object.get_information(
environment, instance_data='true')
if 'instances_cost_dict' in kwargs:
if kwargs['instances_cost_dict'] == 'true':
for organization in organization_list:
return self.cloud_obj.get_cloudability_costs(
)['instances_costs'][organization]
if 'ebs_cost_dict' in kwargs:
if kwargs['ebs_cost_dict'] == 'true':
for organization in organization_list:
return self.cloud_obj.get_cloudability_costs(
)['ebs_costs'][organization]
if 'aws_info_dict' in kwargs:
if kwargs['aws_info_dict'] == 'true':
return self.aws_object.get_information(
environment, 'aws_info_dict')
if 'application_subnets' in kwargs:
if kwargs['application_subnets'] == 'true':
return self.aws_object.get_information(
environment, 'application_subnets')
def get_configuration_data(self, key):
value = self.ah_obj.get_atlas_config_data(self.module, key)
if isinstance(value, dict):
return value[0]
else:
return value
def get_stack_attributes(self, environment=None):
"""
Get stack attributes from config file.
"""
stack_attribute_list = []
stack_attributes_dict = self.ah_obj.get_atlas_config_data(
'cloudability_module', 'stack_attributes')[1]
for attribute, details in stack_attributes_dict.iteritems():
stack_attribute_list.append((attribute, details['editable']))
return (stack_attribute_list, stack_attributes_dict)
def get_attribute_values(self, environment=None):
return self.__get_detailed_instances_cost_dict(environment,
'stack_costs')
def get_status(self, environment=None):
status_information = self.get_configuration_data('status')
cloud_status_dict = {}
organization_list = self.awshelper_obj.get_organizations()
environment_list = []
if environment == None:
env_cost_dict = self.get_information(env_cost_dict='true')
cloud_status_dict = {
environment: ["$" + str(env_cost_dict[environment])]
for environment in env_cost_dict.keys()
}
else:
env_cost_dict = self.get_information(environment,
env_cost_dict='true')
region_vpc_selection = self.aws_object.get_information(
environment, region_vpc_dict='true')
if environment == "uncategorized":
region_list = self.awshelper_obj.get_regions()
for region in region_vpc_selection:
if region == 'east':
cloud_status_dict[region] = [
"$" + str(env_cost_dict[environment])
]
else:
cloud_status_dict[region] = ["$" + "0.0"]
else:
for vpc in ['ame1'
]: #should be changed later to include all vpcs.
cloud_status_dict[vpc] = [
"$" + str(env_cost_dict[environment])
]
return (status_information, cloud_status_dict)
def get_tabs(self, environment=None):
pass
def get_instance_actions(self, environment=None):
pass
def get_environment_actions(self, environment=None):
pass
def get_instance_group_actions(self, environment=None):
pass
def get_stack_actions(self, environment=None):
pass
def get_vpc_actions(self):
pass
def get_action_status(self, json_data, environment=None):
pass
def perform_instance_actions(self, environment=None):
pass
def perform_instancegroup_actions():
pass
def perform_stack_actions():
pass
def perform_vpc_actions(self, json_data):
pass
def perform_environment_actions(self, environment=None):
pass
def get_columns(self, environment=None):
column_list = self.ah_obj.get_atlas_config_data(self.module, 'columns')
column_dict = self.ah_obj.create_nested_defaultdict()
if column_list:
column_dict = self.__get_detailed_instances_cost_dict(
environment, 'instances_cost')
return (column_list, self.ah_obj.defaultdict_to_dict(column_dict))
def get_action_parameters(self, action_type, environment=None):
pass
def load_session(self, request, environment=None):
pass
def save_session(self, request, environment=None):
pass
def get_defaults():
pass
def get_aggregates(self, environment=None):
aggregates = self.ah_obj.get_atlas_config_data(self.module,
'aggregates')
if environment is None:
aggregate_list = [
"$" +
str(self.get_information(ec2_cost_dict='true')['region_zone'])
]
return (aggregates, aggregate_list)
else:
aggregate_dict = collections.defaultdict(dict)
for agg_key in aggregates:
if agg_key == 'cost':
aggregate_dict[agg_key] = self.get_information(
env_cost_dict='true')[environment]
return dict(aggregate_dict)
def refresh_information(self, environment=None):
self.cloud_obj.cache_cloud_costs()
return
def __get_detailed_instances_cost_dict(self, environment, cost_type):
instances_cost_dict = self.get_information(environment,
instances_cost_dict='true')
ebs_cost_dict = self.get_information(environment, ebs_cost_dict='true')
aws_tabs_dict = self.aws_object.get_tabs(environment)[1]
instance_information = self.aws_object.get_information(
environment, instance_data='true')
organization_list = self.awshelper_obj.get_organizations()
(stack_attr_list,
stack_attr_details) = self.get_configuration_data('stack_attributes')
apps_in_environment = self.get_information(environment,
apps_in_environment='true')
application_subnets = self.get_information(environment,
application_subnets='true')
region, vpc, subnet = "", "", ""
instance_cost_column_dict = self.ah_obj.create_nested_defaultdict()
ebs_cost_column_dict = self.ah_obj.create_nested_defaultdict()
stack_cost_dict = self.ah_obj.create_nested_defaultdict()
stack_cost_string_dict = self.ah_obj.create_nested_defaultdict()
name_tag_value, fqdn_tag_value = "", ""
for instance, aws_tabs_dict in aws_tabs_dict.iteritems():
attribute_cost = {}
if 'Name' in aws_tabs_dict['aws_tags']:
name_tag_value = aws_tabs_dict['aws_tags']["Name"]
if 'fqdn' in aws_tabs_dict['aws_tags']:
fqdn_tag_value = aws_tabs_dict['aws_tags']['fqdn']
instance_details = self.ah_obj.get_nested_attribute_values(
instance_information, instance)[1]
region = instance_details[
'region'] if "region" in instance_details else "none"
subnet = instance_details[
'subnet'] if "subnet" in instance_details else "none"
attribute_cost[subnet] = {}
vpc = instance_details[
'vpc'] if "vpc" in instance_details else "none"
attribute_cost[vpc] = {}
attribute_cost[vpc][subnet] = {}
stack = instance_details[
'application'] if "application" in instance_details else "none"
if cost_type == 'instances_cost' or cost_type == 'stack_costs':
if fqdn_tag_value in ebs_cost_dict:
self.storage_cost = ebs_cost_dict[fqdn_tag_value]
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none'][
'instance_attributes'][instance][
'storage_cost'] = "$" + str(
ebs_cost_dict[fqdn_tag_value]) + "/m"
else:
instance_cost_column_dict[vpc]['subnets'][subnet][
'instance_attributes'][instance][
'storage_cost'] = "$" + str(
ebs_cost_dict[fqdn_tag_value]) + "/m"
else:
self.storage_cost = 0.0
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none'][
'instance_attributes'][instance][
'storage_cost'] = "$" + "0.0" + "/m"
else:
instance_cost_column_dict[vpc]['subnets'][subnet][
'instance_attributes'][instance][
'storage_cost'] = "$" + "0.0" + "/m"
if fqdn_tag_value in instances_cost_dict:
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none'][
'instance_attributes'][instance][
'instance_cost'] = "$" + str(
instances_cost_dict[fqdn_tag_value]) + "/m"
else:
self.instance_cost = instances_cost_dict[
fqdn_tag_value]
instance_cost_column_dict[vpc]['subnets'][subnet][
'instance_attributes'][instance][
'instance_cost'] = "$" + str(
instances_cost_dict[fqdn_tag_value]) + "/m"
elif name_tag_value in instances_cost_dict:
self.instance_cost = instances_cost_dict[name_tag_value]
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none'][
'instance_attributes'][instance][
'instance_cost'] = "$" + str(
instances_cost_dict[name_tag_value]) + "/m"
else:
self.instance_cost = instances_cost_dict[
fqdn_tag_value]
instance_cost_column_dict[vpc]['subnets'][subnet][
'instance_attributes'][instance][
'instance_cost'] = "$" + str(
instances_cost_dict[name_tag_value]) + "/m"
else:
if environment == "uncategorized":
instance_cost_column_dict[region]['subnets']['none'][
'instance_attributes'][instance][
'instance_cost'] = "(empty)"
else:
self.instance_cost = 0.0
instance_cost_column_dict[vpc]['subnets'][subnet][
'instance_attributes'][instance][
'instance_cost'] = "(empty)"
if cost_type == 'stack_costs':
attr_cost = 0.0
for attribute in stack_attr_list:
if attribute == 'instance_cost':
attr_cost = self.instance_cost
elif attribute == 'storage_cost':
attr_cost = self.storage_cost
elif attribute == 'total_cost':
attr_cost = self.instance_cost + self.storage_cost
if stack_attr_details[attribute]['stack'] == ['all']:
for apps in apps_in_environment:
if stack == apps:
if not stack_cost_dict[region][vpc][
subnet][stack][attribute]:
stack_cost_dict[region][vpc][subnet][
stack][attribute] = attr_cost
else:
cost = stack_cost_dict[region][vpc][
subnet][stack][attribute]
stack_cost_dict[region][vpc][subnet][
stack][attribute] += attr_cost
else:
for attr_stack in stack_attr_details[attribute][
'stack']:
if attr_stack == stack:
if not stack_cost_dict[region][vpc][
subnet][stack][attribute]:
stack_cost_dict[region][vpc][subnet][
stack][attribute] = attr_cost
else:
stack_cost_dict[region][vpc][subnet][
stack][attribute] += attr_cost
stack_cost_string_dict[region][vpc][subnet][stack][attribute] = \
"$"+str(stack_cost_dict[region][vpc][subnet][stack][attribute])+"/m"
if cost_type == 'stack_costs':
return self.ah_obj.defaultdict_to_dict(stack_cost_string_dict)
if cost_type == 'instances_cost':
return self.ah_obj.defaultdict_to_dict(instance_cost_column_dict)
def get_aws_details(self,regionvpcselected):
organizations = self.awshelper_obj.get_organizations()
aws_details_dict={}
environment_list=[]
aws_details = self.awshelper_obj.get_instances_details()
for organization in organizations:
environment_list = self.awshelper_obj.get_environments(organization)
environment_list.append('uncategorized')
environment_list.append('all')
ah_obj = AtlasHelper()
total_count, total_running, total_stopped=0, 0, 0
region_dict = {}
region_env_list=[]
for environment in environment_list:
if environment != 'all':
env_attributes = self.ah_obj.get_nested_attribute_values(aws_details, environment)[1]
for region in regionvpcselected.keys():
if 'regions' in env_attributes.keys() and region in env_attributes['regions'].keys():
region_keys, region_attributes = ah_obj.get_nested_attribute_values(env_attributes['regions'], region)
if 'count' in region_keys and 'running' in region_keys and 'stopped' in region_keys:
region_count =region_attributes['count']
region_running =region_attributes['running']
region_stopped =region_attributes['stopped']
if environment == "uncategorized":
total_count+=region_count
total_running+=region_running
total_stopped+=region_stopped
region_env_list.append([region, "none", environment, region_count, region_running, region_stopped])
else:
vpc_list = regionvpcselected[region]
for vpc in vpc_list:
if 'vpc' in region_attributes.keys():
vpc_attributes = self.ah_obj.get_nested_attribute_values(region_attributes['vpc'], vpc)[1]
vpc_count = vpc_attributes['count']
vpc_running = vpc_attributes['running']
vpc_stopped = vpc_attributes['stopped']
total_count+=vpc_count
total_running+=vpc_running
total_stopped+=vpc_stopped
region_env_list.append([region, vpc, environment, vpc_count, vpc_running, vpc_stopped])
else:
region_env_list.append([region, vpc, environment, total_count, total_running, total_stopped])
temp_list, total_list = [], []
for count in range(0, len(environment_list)):
total_list.append([environment_list[count],0,0,0])
for index in range(0, len(region_env_list)):
for index1 in range(0, len(total_list)):
if region_env_list[index][2] == total_list[index1][0]:
temp_list = [region_env_list[index][3], region_env_list[index][4], region_env_list[index][5]]
total_list[index1][1]+=temp_list[0]
total_list[index1][2]+=temp_list[1]
total_list[index1][3]+=temp_list[2]
aws_details_dict['total_count'] = total_count
aws_details_dict['total_running'] = total_running
aws_details_dict['total_stopped'] = total_stopped
aws_details_dict['environment_list'] = environment_list
aws_details_dict['region_dict'] = region_dict
aws_details_dict['total_list'] = total_list
return aws_details_dict
class AwsInstanceHelper:
#class to fetch data from aws for atlas
def __init__(self, environment=None):
"""Initialize variables."""
self.ah_obj = AtlasHelper()
self.awshelper_obj = AwsHelper()
self.module = 'aws_module'
self.environment_dict = {}
self.env_subnet_dict = {}
self.env_details_dict = {}
self.environment_groups = self.ah_obj.get_atlas_config_data("global_config_data", "environment_groups")
def get_aws_details(self,regionvpcselected):
organizations = self.awshelper_obj.get_organizations()
aws_details_dict={}
environment_list=[]
aws_details = self.awshelper_obj.get_instances_details()
for organization in organizations:
environment_list = self.awshelper_obj.get_environments(organization)
environment_list.append('uncategorized')
environment_list.append('all')
ah_obj = AtlasHelper()
total_count, total_running, total_stopped=0, 0, 0
region_dict = {}
region_env_list=[]
for environment in environment_list:
if environment != 'all':
env_attributes = self.ah_obj.get_nested_attribute_values(aws_details, environment)[1]
for region in regionvpcselected.keys():
if 'regions' in env_attributes.keys() and region in env_attributes['regions'].keys():
region_keys, region_attributes = ah_obj.get_nested_attribute_values(env_attributes['regions'], region)
if 'count' in region_keys and 'running' in region_keys and 'stopped' in region_keys:
region_count =region_attributes['count']
region_running =region_attributes['running']
region_stopped =region_attributes['stopped']
if environment == "uncategorized":
total_count+=region_count
total_running+=region_running
total_stopped+=region_stopped
region_env_list.append([region, "none", environment, region_count, region_running, region_stopped])
else:
vpc_list = regionvpcselected[region]
for vpc in vpc_list:
if 'vpc' in region_attributes.keys():
vpc_attributes = self.ah_obj.get_nested_attribute_values(region_attributes['vpc'], vpc)[1]
vpc_count = vpc_attributes['count']
vpc_running = vpc_attributes['running']
vpc_stopped = vpc_attributes['stopped']
total_count+=vpc_count
total_running+=vpc_running
total_stopped+=vpc_stopped
region_env_list.append([region, vpc, environment, vpc_count, vpc_running, vpc_stopped])
else:
region_env_list.append([region, vpc, environment, total_count, total_running, total_stopped])
temp_list, total_list = [], []
for count in range(0, len(environment_list)):
total_list.append([environment_list[count],0,0,0])
for index in range(0, len(region_env_list)):
for index1 in range(0, len(total_list)):
if region_env_list[index][2] == total_list[index1][0]:
temp_list = [region_env_list[index][3], region_env_list[index][4], region_env_list[index][5]]
total_list[index1][1]+=temp_list[0]
total_list[index1][2]+=temp_list[1]
total_list[index1][3]+=temp_list[2]
aws_details_dict['total_count'] = total_count
aws_details_dict['total_running'] = total_running
aws_details_dict['total_stopped'] = total_stopped
aws_details_dict['environment_list'] = environment_list
aws_details_dict['region_dict'] = region_dict
aws_details_dict['total_list'] = total_list
return aws_details_dict
def calculate_environment_aggregates(self, environment, env_attributes, region_vpc_dict):
"""
Calculate aggregate values for each environment.
"""
total_list = [0,0,0]
for region in region_vpc_dict.keys():
if region in env_attributes['regions'].keys():
region_attributes = env_attributes['regions'][region]
if environment == 'uncategorized':
region_aggs = [region_attributes['count'],region_attributes['running'], region_attributes['stopped']]
total_list = map(add, total_list, region_aggs)
else:
for vpc in region_vpc_dict[region]:
if vpc and vpc in region_attributes['vpc'].keys():
vpc_properties = self.ah_obj.get_nested_attribute_values(env_attributes['regions'][region], vpc)[1]
vpc_aggregates = [vpc_properties['count'], vpc_properties['running'], vpc_properties['stopped']]
total_list = map(add, total_list, vpc_aggregates)
return total_list
def get_environment_aggregates(self, environment, region_vpc_dict):
"""
Get aggregate status values.
"""
aggregate_list = [0,0,0]
organizations = self.awshelper_obj.get_organizations()
org_attributes = self.ah_obj.get_nested_attribute_values(self.awshelper_obj.get_instances_details(), 'organizations')[1]
for organization in organizations:
if self.environment_groups and environment in self.environment_groups[1].keys():
if environment == 'all':
env_group_for_environment = self.awshelper_obj.get_environments(organization)
else:
env_group_for_environment = self.environment_groups[1][environment]
for env in env_group_for_environment:
env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], env)[1]
env_agg_list = self.calculate_environment_aggregates(env, env_attributes, region_vpc_dict)
aggregate_list = map(add, aggregate_list, env_agg_list)
else:
env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], environment)[1]
aggregate_list = self.calculate_environment_aggregates(environment, env_attributes, region_vpc_dict)
return aggregate_list
def determine_environment_status(self, environment, env_attributes, region_vpc_dict):
vpc_status_dict = {}
for region in region_vpc_dict.keys():
if region not in env_attributes['regions'].keys():
continue
else:
region_attributes = env_attributes['regions'][region]
vpc_status_dict[region] = {'status': []}
if environment == 'uncategorized':
vpc_status_dict[region]['status'] = [region_attributes['count'], region_attributes['running'], region_attributes['stopped']]
else:
for vpc in region_vpc_dict[region]:
if vpc and vpc in region_attributes['vpc'].keys():
vpc_status_dict[vpc] = {'status': []}
vpc_properties = self.ah_obj.get_nested_attribute_values(env_attributes['regions'][region], vpc)[1]
vpc_status_dict[vpc]['status'] = [vpc_properties['count'], vpc_properties['running'], vpc_properties['stopped']]
return vpc_status_dict
def get_environment_status(self, environment, region_vpc_dict):
"""Get environment status."""
ah_obj = AtlasHelper()
vpc_status_dict = {}
organizations = self.awshelper_obj.get_organizations()
org_attributes = self.ah_obj.get_nested_attribute_values(self.awshelper_obj.get_instances_details(), 'organizations')[1]
for organization in organizations:
if self.environment_groups and environment in self.environment_groups[1].keys():
env_status_dict = {}
if environment == 'all':
env_group_for_env = self.awshelper_obj.get_environments(organization)
else:
env_group_for_env = self.environment_groups[1][environment]
for env in env_group_for_env:
env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], env)[1]
env_status_dict = self.determine_environment_status(env, env_attributes, region_vpc_dict)
if not vpc_status_dict:
vpc_status_dict.update(env_status_dict)
else:
for vpc in env_status_dict.keys():
vpc_status_dict[vpc]['status'] = map(add,vpc_status_dict[vpc]['status'],env_status_dict[vpc]['status'])
else:
env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], environment)[1]
vpc_status_dict = self.determine_environment_status(environment, env_attributes, region_vpc_dict)
return vpc_status_dict
def find_env_details(self, environment, env_attributes, region_vpc_dict):
organizations = self.awshelper_obj.get_organizations()
details_dict, env_subnets_dict = {}, {}
environment_subnets = {'vpc':{}}
apps_in_environment, instances_list, application_subnets, env_subnet_list = [], [], [], []
vpc_attribute_dict, aws_info_dict, env_info_dict={}, {}, {}
instances_list,apps_in_environment, env_subnet_list = [], [], []
subnets_with_instances = {} #new addition
instance_keys = self.ah_obj.get_atlas_config_data(self.module, "instance_keys")
for region in region_vpc_dict.keys():
if region not in env_attributes['regions'].keys():
continue
else:
region_attributes = env_attributes['regions'][region]
if environment == 'uncategorized':
vpc_attribute_dict[region] = {"subnets":{"none":{"instance_attributes":{}}}}
for instance, details in region_attributes['uncat_instances'].iteritems():
if details.has_key('instance_attributes') and details['instance_attributes'].has_key('instance_id'):
instance_id = details['instance_attributes']['instance_id']
vpc_attribute_dict[region]["subnets"]["none"]["instance_attributes"][instance_id] = details['instance_attributes']
aws_info_dict[instance_id] = {}
aws_info_dict[instance_id]['aws_information'] = details['aws_information']
aws_info_dict[instance_id]['aws_tags'] = details['aws_tags']
else:
for vpc in region_vpc_dict[region]:
if vpc and vpc in region_attributes['vpc'].keys():
environment_subnets['vpc'][vpc] = {'subnets':[]}
env_subnet_list.extend(self.awshelper_obj.get_subnets_in_environment(region, vpc, environment))
environment_subnets['vpc'][vpc]['subnets'] = env_subnet_list
vpc_attribute_dict[vpc] = {'subnets': {}, 'applications':[]}
vpc_attribute_dict[vpc]['subnets'] = {}
vpc_properties = self.ah_obj.get_nested_attribute_values(env_attributes['regions'][region], vpc)[1]
if 'subnets' in vpc_properties.keys():
for subnet in vpc_properties['subnets'].keys():
instances_details = vpc_properties['subnets'][subnet]['instances']
if instances_details:
#env_subnet_list.append(subnet)
subnets_with_instances[subnet] = []
vpc_attribute_dict[vpc]['subnets'][subnet] = {'instance_attributes': {}}
if subnet not in application_subnets: application_subnets.append(subnet)
for instance, details in instances_details.iteritems():
instance_attributes = details['instance_attributes']
if instance_attributes:
details_key = details['instance_attributes']['instance_id']
vpc_attribute_dict[vpc]['subnets'][subnet]['instance_attributes'][details_key] = instance_attributes
aws_info_dict[details_key] = {}
aws_info_dict[details_key]['aws_information'] = details['aws_information']
aws_info_dict[details_key]['aws_tags'] = details['aws_tags']
instance_application = instance_attributes['application']
if instance_application not in subnets_with_instances[subnet]:
subnets_with_instances[subnet].append(instance_application)
if instance_attributes['application'] not in apps_in_environment:
apps_in_environment.append(instance_attributes['application'])
details_dict['subnets_with_instances'] = subnets_with_instances
details_dict['environment_subnets'] = environment_subnets
details_dict['application_subnets'] = application_subnets
details_dict['vpc_attribute_dict'] = vpc_attribute_dict
details_dict['aws_info_dict'] = aws_info_dict
details_dict['env_subnet_list'] = env_subnet_list
details_dict['apps_in_environment'] = apps_in_environment
return details_dict
def get_environment_details(self, environment, region_vpc_dict):
"""
Get details for a particular environment or environment_group.
"""
organizations = self.awshelper_obj.get_organizations()
org_attributes = self.ah_obj.get_nested_attribute_values(self.awshelper_obj.get_instances_details(), 'organizations')[1]
for organization in organizations:
if self.environment_groups and environment in self.environment_groups[0]:
if environment == 'all':
env_group_for_environment = self.awshelper_obj.get_environments(organization)
else:
env_group_for_environment = self.environment_groups[1][environment]
for env in env_group_for_environment:
env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], env)[1]
environment_details = self.find_env_details(env, env_attributes, region_vpc_dict)
if not self.env_details_dict:
self.env_details_dict.update(environment_details)
else:
generator_obj = self.ah_obj.merge_dictionaries(self.env_details_dict, environment_details)
self.env_details_dict = {key:value for key, value in generator_obj}
else:
env_attributes = self.ah_obj.get_nested_attribute_values(org_attributes[organization], environment)[1]
self.env_details_dict = self.find_env_details(environment, env_attributes, region_vpc_dict)
return self.env_details_dict
def get_column_data(self, environment):
column_dict = {}
if 'vpc_attribute_dict' in self.env_details_dict:
environment_attributes = self.env_details_dict['vpc_attribute_dict']
for vpc, vpc_attrs_dict in environment_attributes.iteritems():
column_dict[vpc] = {'subnets': {}}
if 'subnets' in vpc_attrs_dict:
for subnet , subnet_attributes in vpc_attrs_dict['subnets'].iteritems():
column_dict[vpc]['subnets'][subnet] = {'instance_attributes':[]}
if 'instance_attributes' in subnet_attributes:
column_dict[vpc]['subnets'][subnet]['instance_attributes'] = subnet_attributes['instance_attributes']
return column_dict
class ChefHelper:
def __init__(self):
self.ah_obj = AtlasHelper()
self.awshelper_obj = aws_helper.AwsHelper()
self.module = "chef_module"
self.db_obj = DatabaseHelper()
self.environment_groups = self.ah_obj.get_atlas_config_data("global_config_data", "environment_groups")
self.memcache_var = memcache.Client([self.ah_obj.get_atlas_config_data("global_config_data",
'memcache_server_location')
], debug=0)
self.environment_subnets_details = self.awshelper_obj.get_environment_subnets_details()
try:
base_path = self.ah_obj.get_atlas_config_data("chef_module", 'chef-base-path')
self.api = chef.autoconfigure(base_path)
except Exception as exp_object:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.ah_obj.print_exception("chef_helper.py", "__init__()", exp_object, exc_type, exc_obj, exc_tb)
def get_databag_list(self, databag=''):
#returns a list of all available data bags on the chef_server
data_bag_list = []
try:
data_bags = DataBag(databag,self.api)
for items in data_bags.list(self.api):
data_bag_list.append(items)
if not data_bag_list:
raise Exception, "No Data bags items found"
else:
return data_bag_list
except Exception as exp_object:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.ah_obj.print_exception("chef_helper.py", "get_databag_list()", exp_object, exc_type, exc_obj, exc_tb)
return []
def cache_databag_attributes_foritem(self, databag, item):
"""
Fetch databag attributes from chef server using databag name and item name and cache it locally.
"""
try:
databag_attributes_foritem = self.get_databag_attribute_foritem(databag, item)
self.memcache_var.set("cpdeployment_databag_attrs", databag_attributes_foritem,600)
if databag_attributes_foritem is None:
raise Exception("Databag attributes cannot be obtained from Chef server. Please make sure data is obtained and populate the cache !!!")
if databag_attributes_foritem is not None:
self.memcache_var.set("global_cpdeployment_databag_attributes", databag_attributes_foritem ,86400)
self.memcache_var.disconnect_all()
except Exception as exp_object:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.ah_obj.print_exception("chef_helper.py", "cache_databag_attributes_foritem()", exp_object, exc_type, exc_obj, exc_tb)
self.memcache_var.disconnect_all()
return
def get_chefdbag_attributes_foritem(self, databag, item):
"""
Check in short term cache if not fetch results from global cache
"""
db_attribute_dict = self.memcache_var.get('cpdeployment_databag_attrs')
if db_attribute_dict is None:
db_attribute_dict = self.memcache_var.get('global_cpdeployment_databag_attrs')
if db_attribute_dict is not None:
self.memcache_var.set("cpdeployment_databag_attrs", db_attribute_dict, 600)
self.memcache_var.disconnect_all()
with threading.Lock():
thread = threading.Thread(target=self.cache_databag_attributes_foritem, args=(databag, item))
thread.start()
return db_attribute_dict
def get_databag_attribute_foritem(self, databag, item):
try:
data_bag = DataBag(databag,self.api)
data_bag_item = data_bag[item]
chef_databags = self.ah_obj.get_atlas_config_data("chef_module", 'chef_databags')[0]
chef_databags_info = self.ah_obj.get_atlas_config_data("chef_module", 'chef_databags')[1]
key_list = []
for index in chef_databags:
if databag in chef_databags_info.keys():
for item_index in chef_databags_info[databag]['items'].keys():
if item == item_index:
key_list = chef_databags_info[databag]['items'][item]['keys']
data_bag_attr = {}
data_bag_attr.fromkeys(data_bag_item.keys(), None)
for d_item_key, d_item_values in data_bag_item.iteritems():
if type(d_item_values)== unicode:
if d_item_key in key_list:
data_bag_attr[d_item_key] = {}
data_bag_attr[d_item_key] = d_item_values
break;
elif type(d_item_values) == dict:
data_bag_attr[d_item_key] = {}
for key in key_list:
data_bag_attr[d_item_key][key] = self.ah_obj.get_nested_attribute_values(d_item_values, key)
return data_bag_attr
except Exception as exp_object:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.ah_obj.print_exception("chef_helper.py", "get_databag_attributes()", exp_object, exc_type, exc_obj, exc_tb)
return []
def cache_chef_node_attributes(self):
"""
Fetch node attributes from chef server and cache it locally.
"""
try:
chef_node_attributes_dict = self.get_node_attrs_from_chef()
if chef_node_attributes_dict is None:
raise Exception("Chef node attributes not available from the Chef server. Please make sure the data is available and populate the cache.")
if chef_node_attributes_dict is not None:
self.memcache_var.set("global_chef_node_attributes_cache", chef_node_attributes_dict,86400)
self.memcache_var.disconnect_all()
self.memcache_var.set("chef_node_attr_caches", chef_node_attributes_dict,2*60*60)
self.memcache_var.disconnect_all()
except Exception as exp_object:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.ah_obj.print_exception("chef_helper.py", "cache_chef_node_attributes()", exp_object, exc_type, exc_obj, exc_tb)
self.memcache_var.disconnect_all()
return
def get_node_attributes(self):
node_attribute_dict = self.memcache_var.get('chef_node_attr_caches')
if node_attribute_dict is None:
node_attribute_dict = self.memcache_var.get('global_chef_node_attributes_cache')
self.memcache_var.disconnect_all()
if node_attribute_dict is not None:
self.memcache_var.set('chef_node_attr_caches', node_attribute_dict, 600)
with threading.Lock():
thread = threading.Thread(target=self.cache_chef_node_attributes)
thread.start()
return node_attribute_dict
def get_node_attrs_from_chef(self):
try:
env_subnets_dict = {}
node_attribute_dict = {}
for organization in self.awshelper_obj.get_organizations():
node_attribute_dict = defaultdict(dict)
node_list = Node.list(self.api)
for environment in self.awshelper_obj.get_environments(organization):
for region in self.awshelper_obj.get_regions():
vpc_list = self.awshelper_obj.get_vpc_in_region(region)
if vpc_list:
for vpc in self.awshelper_obj.get_vpc_in_region(region):
env_subnets_dict = self.awshelper_obj.get_env_subnets(organization, region, vpc)
for node in node_list:
node_obj = Node(node, api=self.api)
node_split = self.ah_obj.split_string(node, ["."])
if node_split is None or len(node_split)<=1:
pass
else:
node_subnet = self.ah_obj.split_string(node, ['.'])[1]
for key_tuple, environment in env_subnets_dict.iteritems():
if node_subnet in key_tuple:
environment = env_subnets_dict[key_tuple]
attribute_list = node_obj.attributes
if 'ec2' in attribute_list:
if 'instance_id' in node_obj.attributes.get_dotted('ec2'):
instance_id = node_obj.attributes.get_dotted('ec2.instance_id')
node_attribute_dict[instance_id]['node'] = node
if 'os' in attribute_list:
node_attribute_dict[instance_id]['os']=node_obj['os']
if 'os_version' in attribute_list:
node_attribute_dict[instance_id]['os_version'] = node_obj['os_version']
if 'platform' in attribute_list:
node_attribute_dict[instance_id]['platform'] = node_obj['platform']
if 'platform_version' in attribute_list:
node_attribute_dict[instance_id]['platform_version'] = node_obj['platform_version']
if 'uptime' in attribute_list:
node_attribute_dict[instance_id]['uptime'] = node_obj['uptime']
if 'idletime' in attribute_list:
node_attribute_dict[instance_id]['idletime'] = node_obj['idletime']
return dict(node_attribute_dict)
except Exception as exp_object:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.ah_obj.print_exception("chef_helper.py", "get_node_attrs_from_chef1()", exp_object, exc_type, exc_obj, exc_tb)
return {}
def map_node_databag_attributes(self, node_attributes, databag_attributes, environment):
tabs_info_dict = {}
for instance_id, details in node_attributes.iteritems():
tabs_info_dict[instance_id] = {'chef_information':{}}
tabs_info_dict[instance_id]['chef_information'] = details
for node_attr, node_attr_value in details.iteritems():
tabs_info_dict[instance_id]['chef_information'][node_attr]=node_attr_value
if databag_attributes.has_key(details['node']):
for key, value in databag_attributes.iteritems():
if details['node'] == key:
for attribute, attribute_value in value.iteritems():
tabs_info_dict[instance_id]['chef_information'][attribute] = attribute_value
return tabs_info_dict
def get_values_for_attribute(self, environment, region, vpc, subnet, stack, attribute, details):
attribute_value_dict = {}
if attribute == "owner":
ownership_records = self.db_obj.get_stack_ownership_details(environment, region, vpc, subnet, stack)
if ownership_records is not None:
attribute_value_dict["owner"] = ownership_records.owner
attribute_value_dict["start_time"] = int(ownership_records.start_time.strftime("%s")) * 1000
else:
attribute_value_dict["owner"] = "none"
if attribute == "dbhost" or attribute == "email_override" or attribute == "branch":
attribute_record = self.db_obj.get_stack_attribute_value(environment, region, vpc, subnet, stack, attribute)
if attribute_record is not None:
attribute_value_dict[attribute] = attribute_record.attribute_value
else:
custportal_dbag_attrs = self.memcache_var.get('cpdeployment_databag_attrs');
if custportal_dbag_attrs is not None:
for keys, values in custportal_dbag_attrs.iteritems():
custportal_dbag_attrs_subnet = keys.split(".")[1]
if custportal_dbag_attrs_subnet == subnet and stack in details['stack']:
if attribute == "dbhost":
if values['AWS_DB_HOST']:
attribute_value_dict["dbhost"] = values['AWS_DB_HOST']
else:
attribute_value_dict["dbhost"] = "none"
if attribute == "email_override":
if values['EMAIL_OVERRIDE'] and values['EMAIL_OVERRIDE'] is not None:
attribute_value_dict["email_override"] = values['EMAIL_OVERRIDE']
if values['EMAIL_OVERRIDE'] == "none":
pass
if attribute == "branch":
if values["branch"]:
attribute_value_dict["branch"] = values['branch']
else:
"none"
return attribute_value_dict
def stack_attribute_values(self, request, environment, region_vpc_dict):
"""
Get attributes and values for each stack.
"""
stack_attribute_dict = collections.defaultdict(dict)
awsmodule_obj = AwsModule(request, environment)
(stack_attr_list, stack_attr_details) = stack_attributes = self.ah_obj.get_atlas_config_data(self.module, 'stack_attributes')
application_subnets = awsmodule_obj.get_information(environment, application_subnets='true')
apps_in_environment = awsmodule_obj.get_information(environment, apps_in_environment='true')
if application_subnets is not None and apps_in_environment is not None:
for region, vpc_list in region_vpc_dict.iteritems():
stack_attribute_dict[region] = {}
if vpc_list is not None:
for vpc in vpc_list:
stack_attribute_dict[region][vpc] = {}
for subnet in application_subnets:
stack_attribute_dict[region][vpc][subnet] = {}
for stack in apps_in_environment:
stack_attribute_dict[region][vpc][subnet][stack] = {}
for attribute in stack_attr_list:
details = stack_attr_details[attribute]
stack_attribute_dict[region][vpc][subnet][stack].update(self.get_values_for_attribute(environment, region, vpc, subnet, stack, attribute, details))
return dict(stack_attribute_dict)
def get_stack_attribute_values(self, request, environment, region_vpc_dict):
"""
Get stack attribute values for environment or environment groups.
"""
stack_attribute_dict = {}
stack_attribute_dict = self.stack_attribute_values(request, environment, region_vpc_dict)
return stack_attribute_dict
def get_stack_attributes(self, environment):
"""
Get stack attributes from config file.
"""
stack_attribute_list = []
stack_attributes_dict = self.ah_obj.get_atlas_config_data('chef_module', 'stack_attributes')[1]
for attribute, details in stack_attributes_dict.iteritems():
stack_attribute_list.append((attribute, details['editable']))
return(stack_attribute_list, stack_attributes_dict)