class CheckSIT(object): def __init__(self, configs_directory=None, session=None): self.sit_helper = SITHelper(configs_directory) self.ROLES = self.sit_helper.get_roles() self.SIT_CONFIGS = self.sit_helper.get_configs('sit') self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere') self.cf_helper = CFHelper(configs_directory=configs_directory, session=session) def check_configs_are_set(self): missing_configs = [config for config, value in self.SIT_CONFIGS.iteritems() if value is None] if missing_configs: Log.error('The following configs are not set: {0}'.format(missing_configs)) def check_roles_file(self): if self.ROLES < 1: Log.error('roles.yml file is not setup properly. You require at least one role to test') def check_roles_not_empty(self): empty_roles = [role for role in self.ROLES if not self.sit_helper.get_states_for_role(role)] if empty_roles: Log.error('The following servers are missing roles inside of roles.yml file: {0}'.format(empty_roles)) def check_stack_exists(self): stack_name = self.TROPOSPHERE_CONFIGS['stack_name'] if not self.cf_helper.get_stack_info(stack_name): Log.error('Stack "{0}" does not exist. Please run setup_troposphere'.format(stack_name)) def run(self): self.check_configs_are_set() self.check_roles_file() self.check_roles_not_empty() self.check_stack_exists()
def __init__(self, configs_directory=None, session=None): self.sit_helper = SITHelper(configs_directory) self.ROLES = self.sit_helper.get_roles() self.SIT_CONFIGS = self.sit_helper.get_configs('sit') self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere') self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
def __init__(self, job_name=None, build_number=None, master_ip=None, configs_directory=None, session=None): self.check_sit(configs_directory=configs_directory, session=session) sit_helper = SITHelper(configs_directory) self.configs_directory = configs_directory sit_configs = sit_helper.get_configs('sit') troposphere_configs = sit_helper.get_configs('troposphere') self.PROFILE = sit_configs['profile_name'] self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs['autoscaling_group_name'] self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name'] self.STACK_NAME = troposphere_configs['stack_name'] self.ROLES = sit_helper.get_roles() self.ATTEMPT_LIMIT = sit_configs['attempt_limit'] self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit'] self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size'] self.SAVE_LOGS = sit_configs['save_logs'] self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir'] self.REDIS_HOST = sit_configs.get('redis_host', None) self.cf_helper = CFHelper(configs_directory=configs_directory, session=session) self.family = self.join_items([job_name, build_number]) self.master_ip = master_ip self.is_build_successful = True self.instance_was_launched = False self.instance_was_terminated = False self.init_boto_clients(session=session) self.cluster = self.get_cluster() self.autoscaling_group = self.get_autoscaling_group() self.running_task_ids = []
def __init__(self, job_name=None, build_number=None, master_ip=None, configs_directory=None, session=None): self.check_sit(configs_directory=configs_directory, session=session) sit_helper = SITHelper(configs_directory) self.configs_directory = configs_directory sit_configs = sit_helper.get_configs('sit') troposphere_configs = sit_helper.get_configs('troposphere') self.PROFILE = sit_configs['profile_name'] self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs[ 'autoscaling_group_name'] self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name'] self.STACK_NAME = troposphere_configs['stack_name'] self.ROLES = sit_helper.get_roles() self.ATTEMPT_LIMIT = sit_configs['attempt_limit'] self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit'] self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size'] self.SAVE_LOGS = sit_configs['save_logs'] self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir'] self.REDIS_HOST = sit_configs.get('redis_host', None) self.cf_helper = CFHelper(configs_directory=configs_directory, session=session) self.family = self.join_items([job_name, build_number]) self.master_ip = master_ip self.is_build_successful = True self.instance_was_launched = False self.instance_was_terminated = False self.init_boto_clients(session=session) self.cluster = self.get_cluster() self.autoscaling_group = self.get_autoscaling_group() self.running_task_ids = []
def __init__(self, configs_directory='configs', session=None): self.configs = SITHelper(configs_directory).get_configs('troposphere') self.STACK_NAME = self.configs['stack_name'] self.KEY_NAME = self.configs['key_name'] self.AMI_ID = self.configs['ami_id'] self.AMI_URL = self.configs['ami_url'] self.SECURITY_GROUPS = self.configs['security_groups'] self.SUBNET = self.configs['subnet'] self.ec2_helper = EC2Helper(configs_directory=configs_directory, session=session)
class Container(object): def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local', redis_host=None): self.sit_helper = SITHelper(configs_directory) sit_configs = self.sit_helper.get_configs('sit') self.MEMORY = sit_configs['container_memory'] self.MEMORY_RESERVATION = sit_configs.get( 'container_memory_reservation', self.MEMORY) self.CPU = sit_configs['container_cpu'] self.IMAGE = sit_configs['container_image'] self.family = family self.role = role self.master_ip = master_ip self.env = env self.redis_host = self.generate_redis_host(master_ip, redis_host) def get_container_definitions(self): return { "name": self.family, "memory": self.MEMORY, "cpu": self.CPU, "image": self.IMAGE, "environment": self.get_environment_variables(), "memoryReservation": self.MEMORY_RESERVATION } def get_environment_variables(self): environment_variables = list() environment_variables.append( Container.get_environment_dictionary( 'roles', self.get_role_states(self.role))) environment_variables.append( Container.get_environment_dictionary('env', self.env)) environment_variables.append( Container.get_environment_dictionary('master', self.master_ip)) environment_variables.append( Container.get_environment_dictionary('minion_id', self.family)) environment_variables.append( Container.get_environment_dictionary('redis_host', self.redis_host)) return environment_variables @staticmethod def generate_redis_host(master_ip, redis_host): return redis_host if redis_host else master_ip @staticmethod def get_environment_dictionary(name, value): return {"name": name, "value": value} def get_role_states(self, role): return ','.join(self.sit_helper.get_states_for_role(role))
class CheckSIT(object): def __init__(self, configs_directory=None, session=None): self.sit_helper = SITHelper(configs_directory) self.ROLES = self.sit_helper.get_roles() self.SIT_CONFIGS = self.sit_helper.get_configs('sit') self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere') self.cf_helper = CFHelper(configs_directory=configs_directory, session=session) def check_configs_are_set(self): missing_configs = [ config for config, value in self.SIT_CONFIGS.iteritems() if value is None ] if missing_configs: Log.error('The following configs are not set: {0}'.format( missing_configs)) def check_roles_file(self): if self.ROLES < 1: Log.error( 'roles.yml file is not setup properly. You require at least one role to test' ) def check_roles_not_empty(self): empty_roles = [ role for role in self.ROLES if not self.sit_helper.get_states_for_role(role) ] if empty_roles: Log.error( 'The following servers are missing roles inside of roles.yml file: {0}' .format(empty_roles)) def check_stack_exists(self): stack_name = self.TROPOSPHERE_CONFIGS['stack_name'] if not self.cf_helper.get_stack_info(stack_name): Log.error( 'Stack "{0}" does not exist. Please run setup_troposphere'. format(stack_name)) def run(self): self.check_configs_are_set() self.check_roles_file() self.check_roles_not_empty() self.check_stack_exists()
class SitHelperTest(unittest.TestCase): def setUp(self): self.sit_helper = SITHelper('tests/helpers/configs') self.sit_helper_empty = SITHelper('tests/helpers/empty_configs') def test_get_custom_user_data(self): self.assertEquals(self.sit_helper.get_custom_user_data(), "'test'\n") def test_get_roles(self): self.assertEquals(self.sit_helper.get_roles(), ['lb', 'php']) self.assertEquals(self.sit_helper.get_roles(), ['lb', 'php']) self.assertEquals(self.sit_helper_empty.get_roles(), False) def test_get_states_for_role(self): self.assertEquals(self.sit_helper.get_states_for_role('php'), ['apache', 'server']) self.assertEquals(self.sit_helper.get_states_for_role('lb'), ['lb', 'server']) self.assertEquals( self.sit_helper.get_states_for_role('saltmaster'), "Failed to find state list for role: saltmaster. error: 'saltmaster'" ) def test_get_configs(self): self.assertEquals( self.sit_helper.get_configs('roles'), { 'lb': { 'priority': 1, 'subroles': ['lb', 'server'] }, 'php': { 'priority': 2, 'subroles': ['apache', 'server'] } })
class Container(object): def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local', redis_host=None): self.sit_helper = SITHelper(configs_directory) sit_configs = self.sit_helper.get_configs('sit') self.MEMORY = sit_configs['container_memory'] self.MEMORY_RESERVATION = sit_configs.get('container_memory_reservation', self.MEMORY) self.CPU = sit_configs['container_cpu'] self.IMAGE = sit_configs['container_image'] self.family = family self.role = role self.master_ip = master_ip self.env = env self.redis_host = self.generate_redis_host(master_ip, redis_host) def get_container_definitions(self): return { "name": self.family, "memory": self.MEMORY, "cpu": self.CPU, "image": self.IMAGE, "environment": self.get_environment_variables(), "memoryReservation": self.MEMORY_RESERVATION } def get_environment_variables(self): environment_variables = list() environment_variables.append(Container.get_environment_dictionary('roles', self.get_role_states(self.role))) environment_variables.append(Container.get_environment_dictionary('env', self.env)) environment_variables.append(Container.get_environment_dictionary('master', self.master_ip)) environment_variables.append(Container.get_environment_dictionary('minion_id', self.family)) environment_variables.append(Container.get_environment_dictionary('redis_host', self.redis_host)) return environment_variables @staticmethod def generate_redis_host(master_ip, redis_host): return redis_host if redis_host else master_ip @staticmethod def get_environment_dictionary(name, value): return {"name": name, "value": value} def get_role_states(self, role): return ','.join(self.sit_helper.get_states_for_role(role))
def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local'): self.sit_helper = SITHelper(configs_directory) sit_configs = self.sit_helper.get_configs('sit') self.MEMORY = sit_configs['container_memory'] self.CPU = sit_configs['container_cpu'] self.IMAGE = sit_configs['container_image'] self.family = family self.role = role self.master_ip = master_ip self.env = env
def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local', redis_host=None): self.sit_helper = SITHelper(configs_directory) sit_configs = self.sit_helper.get_configs('sit') self.MEMORY = sit_configs['container_memory'] self.MEMORY_RESERVATION = sit_configs.get( 'container_memory_reservation', self.MEMORY) self.CPU = sit_configs['container_cpu'] self.IMAGE = sit_configs['container_image'] self.family = family self.role = role self.master_ip = master_ip self.env = env self.redis_host = self.generate_redis_host(master_ip, redis_host)
def set_values(self, configs_location): self.check_sit = CheckSIT(configs_directory=configs_location, session=self.session) self.check_sit.SIT_HELPER = SITHelper( configs_directory=configs_location) self.check_sit.ROLES = self.check_sit.SIT_HELPER.get_roles() self.check_sit.SIT_CONFIGS = self.check_sit.SIT_HELPER.get_configs( 'sit') self.check_sit.TROPOSPHERE_CONFIGS = self.check_sit.SIT_HELPER.get_configs( 'troposphere')
def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local', redis_host=None): self.sit_helper = SITHelper(configs_directory) sit_configs = self.sit_helper.get_configs('sit') self.MEMORY = sit_configs['container_memory'] self.MEMORY_RESERVATION = sit_configs.get('container_memory_reservation', self.MEMORY) self.CPU = sit_configs['container_cpu'] self.IMAGE = sit_configs['container_image'] self.family = family self.role = role self.master_ip = master_ip self.env = env self.redis_host = self.generate_redis_host(master_ip, redis_host)
def __init__(self, configs_directory='configs', session=None): troposphere_configs = SITHelper(configs_directory).get_configs( 'troposphere') self.STACK_NAME = troposphere_configs['stack_name'] self.TAG_VALUE = troposphere_configs['tag_value'] self.AMI_ID = troposphere_configs['ami_id'] self.SUBNET = troposphere_configs['subnet'] self.SECURITY_GROUPS = troposphere_configs['security_groups'] self.KEY_NAME = troposphere_configs['key_name'] self.AMI_URL = troposphere_configs['ami_url'] self.cf_helper = CFHelper(configs_directory=configs_directory, session=session) self.sit_template = SITTemplate(configs_directory) self.template = self.sit_template.template self.template_json = self.template.to_json()
class SITTemplateHelper(object): def __init__(self, configs_directory='configs', session=None): self.configs = SITHelper(configs_directory).get_configs('troposphere') self.STACK_NAME = self.configs['stack_name'] self.KEY_NAME = self.configs['key_name'] self.AMI_ID = self.configs['ami_id'] self.AMI_URL = self.configs['ami_url'] self.SECURITY_GROUPS = self.configs['security_groups'] self.SUBNET = self.configs['subnet'] self.ec2_helper = EC2Helper(configs_directory=configs_directory, session=session) def validate_configs(self): logging.info('Validating configs') missing_configs = [config for config, value in self.configs.iteritems() if value is None] if missing_configs: Log.error('The following configs have not been added in configs/troposphere.yml: {0}'.format(missing_configs)) def validate_aws_resources(self): logging.info('Validating AWS resources exist') if not self.ec2_helper.describe_key_pairs([self.KEY_NAME]): self.not_found_error('Key pair', self.KEY_NAME) if not self.ec2_helper.describe_images([self.AMI_ID]): self.open_url(self.AMI_URL) self.not_found_error('AMI ID', self.KEY_NAME) if not self.ec2_helper.describe_security_groups(self.SECURITY_GROUPS): self.not_found_error('Security Groups', self.SECURITY_GROUPS) if not self.ec2_helper.describe_subnets([self.SUBNET]): self.not_found_error('Subnet', self.SUBNET) def open_url(self, url): subprocess.call(['open', url]) def not_found_error(self, resource_name, resource): Log.error('{0} "{1}" not found. Please update configs/troposphere.yml'.format(resource_name, resource)) def validate(self): self.validate_configs() self.validate_aws_resources()
def __init__(self, configs_directory='configs'): configs = SITHelper(configs_directory).get_configs('troposphere') self.TEMPLATE_DESCRIPTION = configs["template_description"] self.INSTANCE_TYPE = configs["instance_type"] self.SECURITY_GROUPS = configs["security_groups"] self.KEY_NAME = configs["key_name"] self.TAG_KEY = configs["tag_key"] self.TAG_VALUE = configs["tag_value"] self.AMI_ID = configs['ami_id'] self.EBS_VOLUME_SIZE = configs['ebs_volume_size'] self.EBS_DEVICE_NAME = configs['ebs_device_name'] self.MAX_SIZE = configs['max_size'] self.MIN_SIZE = configs['min_size'] self.SUBNET = configs['subnet'] self.CLUSTER_NAME = configs['cluster_name'] self.AUTOSCALING_GROUP_NAME = configs['autoscaling_group_name'] self.LAUNCH_CONFIGURATION_NAME = configs['launch_configuration_name'] self.AUTOSCALE_UP_ALARMS = configs['autoscale_up_alarms'] self.AUTOSCALE_DOWN_ALARMS = configs['autoscale_down_alarms'] self.ECS_TASK_CLEANUP_WAIT = configs['ecs_task_cleanup_wait_duration'] self.template = Template() self.user_data = UserData(configs_directory) self.init_template()
class SitHelperTest(unittest.TestCase): def setUp(self): self.sit_helper = SITHelper('tests/helpers/configs') self.sit_helper_empty = SITHelper('tests/helpers/empty_configs') def test_get_custom_user_data(self): self.assertEquals(self.sit_helper.get_custom_user_data(), "'test'\n") def test_get_roles(self): self.assertEquals(self.sit_helper.get_roles(), ['php', 'lb']) self.assertEquals(self.sit_helper.get_roles(), ['php', 'lb']) self.assertEquals(self.sit_helper_empty.get_roles(), False) def test_get_states_for_role(self): self.assertEquals(self.sit_helper.get_states_for_role('php'), ['apache', 'server']) self.assertEquals(self.sit_helper.get_states_for_role('lb'), ['lb', 'server']) self.assertEquals(self.sit_helper.get_states_for_role('saltmaster'), "Failed to find state list for role: saltmaster. error: 'saltmaster'") def test_get_configs(self): self.assertEquals(self.sit_helper.get_configs('roles'), {'php': ['apache', 'server'], 'lb': ['lb', 'server']})
def setUp(self): self.sit_helper = SITHelper('tests/helpers/configs') self.sit_helper_empty = SITHelper('tests/helpers/empty_configs')
def __init__(self, configs_directory='tests/sit/configs'): sit_helper = SITHelper(configs_directory) CONFIGS = sit_helper.get_configs('troposphere') self.LAUNCH_CONFIGURATION_NAME = CONFIGS['launch_configuration_name'] self.AUTOSCALING_GROUP_NAME = CONFIGS['autoscaling_group_name'] self.CUSTOM_USER_DATA = sit_helper.get_custom_user_data()
def __init__(self, configs_directory='configs', session=None): if session is None: sit_configs = SITHelper(configs_directory).get_configs('sit') session = Session(profile_name=sit_configs['profile_name']) self.ec2_client = session.client('ec2')
def __init__(self, configs_directory=None, session=None): if session is None: sit_configs = SITHelper(configs_directory).get_configs('sit') session = Session(profile_name=sit_configs['profile_name']) self.cf_client = session.client('cloudformation')
def __init__(self, configs_directory="tests/sit/configs"): sit_helper = SITHelper(configs_directory) CONFIGS = sit_helper.get_configs("troposphere") self.LAUNCH_CONFIGURATION_NAME = CONFIGS["launch_configuration_name"] self.AUTOSCALING_GROUP_NAME = CONFIGS["autoscaling_group_name"] self.CUSTOM_USER_DATA = sit_helper.get_custom_user_data()