class CheckSIT(object):
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
def check_configs_are_set(self):
missing_configs = [config for config, value in self.SIT_CONFIGS.iteritems() if value is None]
if missing_configs:
Log.error('The following configs are not set: {0}'.format(missing_configs))
def check_roles_file(self):
if self.ROLES < 1:
Log.error('roles.yml file is not setup properly. You require at least one role to test')
def check_roles_not_empty(self):
empty_roles = [role for role in self.ROLES if not self.sit_helper.get_states_for_role(role)]
if empty_roles:
Log.error('The following servers are missing roles inside of roles.yml file: {0}'.format(empty_roles))
def check_stack_exists(self):
stack_name = self.TROPOSPHERE_CONFIGS['stack_name']
if not self.cf_helper.get_stack_info(stack_name):
Log.error('Stack "{0}" does not exist. Please run setup_troposphere'.format(stack_name))
def run(self):
self.check_configs_are_set()
self.check_roles_file()
self.check_roles_not_empty()
self.check_stack_exists()
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
def __init__(self, job_name=None, build_number=None, master_ip=None, configs_directory=None, session=None):
self.check_sit(configs_directory=configs_directory, session=session)
sit_helper = SITHelper(configs_directory)
self.configs_directory = configs_directory
sit_configs = sit_helper.get_configs('sit')
troposphere_configs = sit_helper.get_configs('troposphere')
self.PROFILE = sit_configs['profile_name']
self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs['autoscaling_group_name']
self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name']
self.STACK_NAME = troposphere_configs['stack_name']
self.ROLES = sit_helper.get_roles()
self.ATTEMPT_LIMIT = sit_configs['attempt_limit']
self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit']
self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size']
self.SAVE_LOGS = sit_configs['save_logs']
self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir']
self.REDIS_HOST = sit_configs.get('redis_host', None)
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
self.family = self.join_items([job_name, build_number])
self.master_ip = master_ip
self.is_build_successful = True
self.instance_was_launched = False
self.instance_was_terminated = False
self.init_boto_clients(session=session)
self.cluster = self.get_cluster()
self.autoscaling_group = self.get_autoscaling_group()
self.running_task_ids = []
def __init__(self,
job_name=None,
build_number=None,
master_ip=None,
configs_directory=None,
session=None):
self.check_sit(configs_directory=configs_directory, session=session)
sit_helper = SITHelper(configs_directory)
self.configs_directory = configs_directory
sit_configs = sit_helper.get_configs('sit')
troposphere_configs = sit_helper.get_configs('troposphere')
self.PROFILE = sit_configs['profile_name']
self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs[
'autoscaling_group_name']
self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name']
self.STACK_NAME = troposphere_configs['stack_name']
self.ROLES = sit_helper.get_roles()
self.ATTEMPT_LIMIT = sit_configs['attempt_limit']
self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit']
self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size']
self.SAVE_LOGS = sit_configs['save_logs']
self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir']
self.REDIS_HOST = sit_configs.get('redis_host', None)
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
self.family = self.join_items([job_name, build_number])
self.master_ip = master_ip
self.is_build_successful = True
self.instance_was_launched = False
self.instance_was_terminated = False
self.init_boto_clients(session=session)
self.cluster = self.get_cluster()
self.autoscaling_group = self.get_autoscaling_group()
self.running_task_ids = []
def __init__(self, configs_directory='configs', session=None):
self.configs = SITHelper(configs_directory).get_configs('troposphere')
self.STACK_NAME = self.configs['stack_name']
self.KEY_NAME = self.configs['key_name']
self.AMI_ID = self.configs['ami_id']
self.AMI_URL = self.configs['ami_url']
self.SECURITY_GROUPS = self.configs['security_groups']
self.SUBNET = self.configs['subnet']
self.ec2_helper = EC2Helper(configs_directory=configs_directory, session=session)
class Container(object):
def __init__(self,
configs_directory=None,
family=None,
role=None,
master_ip=None,
env='local',
redis_host=None):
self.sit_helper = SITHelper(configs_directory)
sit_configs = self.sit_helper.get_configs('sit')
self.MEMORY = sit_configs['container_memory']
self.MEMORY_RESERVATION = sit_configs.get(
'container_memory_reservation', self.MEMORY)
self.CPU = sit_configs['container_cpu']
self.IMAGE = sit_configs['container_image']
self.family = family
self.role = role
self.master_ip = master_ip
self.env = env
self.redis_host = self.generate_redis_host(master_ip, redis_host)
def get_container_definitions(self):
return {
"name": self.family,
"memory": self.MEMORY,
"cpu": self.CPU,
"image": self.IMAGE,
"environment": self.get_environment_variables(),
"memoryReservation": self.MEMORY_RESERVATION
}
def get_environment_variables(self):
environment_variables = list()
environment_variables.append(
Container.get_environment_dictionary(
'roles', self.get_role_states(self.role)))
environment_variables.append(
Container.get_environment_dictionary('env', self.env))
environment_variables.append(
Container.get_environment_dictionary('master', self.master_ip))
environment_variables.append(
Container.get_environment_dictionary('minion_id', self.family))
environment_variables.append(
Container.get_environment_dictionary('redis_host',
self.redis_host))
return environment_variables
@staticmethod
def generate_redis_host(master_ip, redis_host):
return redis_host if redis_host else master_ip
@staticmethod
def get_environment_dictionary(name, value):
return {"name": name, "value": value}
def get_role_states(self, role):
return ','.join(self.sit_helper.get_states_for_role(role))
class CheckSIT(object):
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
def check_configs_are_set(self):
missing_configs = [
config for config, value in self.SIT_CONFIGS.iteritems()
if value is None
]
if missing_configs:
Log.error('The following configs are not set: {0}'.format(
missing_configs))
def check_roles_file(self):
if self.ROLES < 1:
Log.error(
'roles.yml file is not setup properly. You require at least one role to test'
)
def check_roles_not_empty(self):
empty_roles = [
role for role in self.ROLES
if not self.sit_helper.get_states_for_role(role)
]
if empty_roles:
Log.error(
'The following servers are missing roles inside of roles.yml file: {0}'
.format(empty_roles))
def check_stack_exists(self):
stack_name = self.TROPOSPHERE_CONFIGS['stack_name']
if not self.cf_helper.get_stack_info(stack_name):
Log.error(
'Stack "{0}" does not exist. Please run setup_troposphere'.
format(stack_name))
def run(self):
self.check_configs_are_set()
self.check_roles_file()
self.check_roles_not_empty()
self.check_stack_exists()
class SitHelperTest(unittest.TestCase):
def setUp(self):
self.sit_helper = SITHelper('tests/helpers/configs')
self.sit_helper_empty = SITHelper('tests/helpers/empty_configs')
def test_get_custom_user_data(self):
self.assertEquals(self.sit_helper.get_custom_user_data(), "'test'\n")
def test_get_roles(self):
self.assertEquals(self.sit_helper.get_roles(), ['lb', 'php'])
self.assertEquals(self.sit_helper.get_roles(), ['lb', 'php'])
self.assertEquals(self.sit_helper_empty.get_roles(), False)
def test_get_states_for_role(self):
self.assertEquals(self.sit_helper.get_states_for_role('php'),
['apache', 'server'])
self.assertEquals(self.sit_helper.get_states_for_role('lb'),
['lb', 'server'])
self.assertEquals(
self.sit_helper.get_states_for_role('saltmaster'),
"Failed to find state list for role: saltmaster. error: 'saltmaster'"
)
def test_get_configs(self):
self.assertEquals(
self.sit_helper.get_configs('roles'), {
'lb': {
'priority': 1,
'subroles': ['lb', 'server']
},
'php': {
'priority': 2,
'subroles': ['apache', 'server']
}
})
class Container(object):
def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local', redis_host=None):
self.sit_helper = SITHelper(configs_directory)
sit_configs = self.sit_helper.get_configs('sit')
self.MEMORY = sit_configs['container_memory']
self.MEMORY_RESERVATION = sit_configs.get('container_memory_reservation', self.MEMORY)
self.CPU = sit_configs['container_cpu']
self.IMAGE = sit_configs['container_image']
self.family = family
self.role = role
self.master_ip = master_ip
self.env = env
self.redis_host = self.generate_redis_host(master_ip, redis_host)
def get_container_definitions(self):
return {
"name": self.family,
"memory": self.MEMORY,
"cpu": self.CPU,
"image": self.IMAGE,
"environment": self.get_environment_variables(),
"memoryReservation": self.MEMORY_RESERVATION
}
def get_environment_variables(self):
environment_variables = list()
environment_variables.append(Container.get_environment_dictionary('roles', self.get_role_states(self.role)))
environment_variables.append(Container.get_environment_dictionary('env', self.env))
environment_variables.append(Container.get_environment_dictionary('master', self.master_ip))
environment_variables.append(Container.get_environment_dictionary('minion_id', self.family))
environment_variables.append(Container.get_environment_dictionary('redis_host', self.redis_host))
return environment_variables
@staticmethod
def generate_redis_host(master_ip, redis_host):
return redis_host if redis_host else master_ip
@staticmethod
def get_environment_dictionary(name, value):
return {"name": name, "value": value}
def get_role_states(self, role):
return ','.join(self.sit_helper.get_states_for_role(role))
def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local'):
self.sit_helper = SITHelper(configs_directory)
sit_configs = self.sit_helper.get_configs('sit')
self.MEMORY = sit_configs['container_memory']
self.CPU = sit_configs['container_cpu']
self.IMAGE = sit_configs['container_image']
self.family = family
self.role = role
self.master_ip = master_ip
self.env = env
def __init__(self,
configs_directory=None,
family=None,
role=None,
master_ip=None,
env='local',
redis_host=None):
self.sit_helper = SITHelper(configs_directory)
sit_configs = self.sit_helper.get_configs('sit')
self.MEMORY = sit_configs['container_memory']
self.MEMORY_RESERVATION = sit_configs.get(
'container_memory_reservation', self.MEMORY)
self.CPU = sit_configs['container_cpu']
self.IMAGE = sit_configs['container_image']
self.family = family
self.role = role
self.master_ip = master_ip
self.env = env
self.redis_host = self.generate_redis_host(master_ip, redis_host)
def set_values(self, configs_location):
self.check_sit = CheckSIT(configs_directory=configs_location,
session=self.session)
self.check_sit.SIT_HELPER = SITHelper(
configs_directory=configs_location)
self.check_sit.ROLES = self.check_sit.SIT_HELPER.get_roles()
self.check_sit.SIT_CONFIGS = self.check_sit.SIT_HELPER.get_configs(
'sit')
self.check_sit.TROPOSPHERE_CONFIGS = self.check_sit.SIT_HELPER.get_configs(
'troposphere')
def __init__(self, configs_directory=None, family=None, role=None, master_ip=None, env='local', redis_host=None):
self.sit_helper = SITHelper(configs_directory)
sit_configs = self.sit_helper.get_configs('sit')
self.MEMORY = sit_configs['container_memory']
self.MEMORY_RESERVATION = sit_configs.get('container_memory_reservation', self.MEMORY)
self.CPU = sit_configs['container_cpu']
self.IMAGE = sit_configs['container_image']
self.family = family
self.role = role
self.master_ip = master_ip
self.env = env
self.redis_host = self.generate_redis_host(master_ip, redis_host)
def __init__(self, configs_directory='configs', session=None):
troposphere_configs = SITHelper(configs_directory).get_configs(
'troposphere')
self.STACK_NAME = troposphere_configs['stack_name']
self.TAG_VALUE = troposphere_configs['tag_value']
self.AMI_ID = troposphere_configs['ami_id']
self.SUBNET = troposphere_configs['subnet']
self.SECURITY_GROUPS = troposphere_configs['security_groups']
self.KEY_NAME = troposphere_configs['key_name']
self.AMI_URL = troposphere_configs['ami_url']
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
self.sit_template = SITTemplate(configs_directory)
self.template = self.sit_template.template
self.template_json = self.template.to_json()
class SITTemplateHelper(object):
def __init__(self, configs_directory='configs', session=None):
self.configs = SITHelper(configs_directory).get_configs('troposphere')
self.STACK_NAME = self.configs['stack_name']
self.KEY_NAME = self.configs['key_name']
self.AMI_ID = self.configs['ami_id']
self.AMI_URL = self.configs['ami_url']
self.SECURITY_GROUPS = self.configs['security_groups']
self.SUBNET = self.configs['subnet']
self.ec2_helper = EC2Helper(configs_directory=configs_directory, session=session)
def validate_configs(self):
logging.info('Validating configs')
missing_configs = [config for config, value in self.configs.iteritems() if value is None]
if missing_configs:
Log.error('The following configs have not been added in configs/troposphere.yml: {0}'.format(missing_configs))
def validate_aws_resources(self):
logging.info('Validating AWS resources exist')
if not self.ec2_helper.describe_key_pairs([self.KEY_NAME]):
self.not_found_error('Key pair', self.KEY_NAME)
if not self.ec2_helper.describe_images([self.AMI_ID]):
self.open_url(self.AMI_URL)
self.not_found_error('AMI ID', self.KEY_NAME)
if not self.ec2_helper.describe_security_groups(self.SECURITY_GROUPS):
self.not_found_error('Security Groups', self.SECURITY_GROUPS)
if not self.ec2_helper.describe_subnets([self.SUBNET]):
self.not_found_error('Subnet', self.SUBNET)
def open_url(self, url):
subprocess.call(['open', url])
def not_found_error(self, resource_name, resource):
Log.error('{0} "{1}" not found. Please update configs/troposphere.yml'.format(resource_name, resource))
def validate(self):
self.validate_configs()
self.validate_aws_resources()
def __init__(self, configs_directory='configs'):
configs = SITHelper(configs_directory).get_configs('troposphere')
self.TEMPLATE_DESCRIPTION = configs["template_description"]
self.INSTANCE_TYPE = configs["instance_type"]
self.SECURITY_GROUPS = configs["security_groups"]
self.KEY_NAME = configs["key_name"]
self.TAG_KEY = configs["tag_key"]
self.TAG_VALUE = configs["tag_value"]
self.AMI_ID = configs['ami_id']
self.EBS_VOLUME_SIZE = configs['ebs_volume_size']
self.EBS_DEVICE_NAME = configs['ebs_device_name']
self.MAX_SIZE = configs['max_size']
self.MIN_SIZE = configs['min_size']
self.SUBNET = configs['subnet']
self.CLUSTER_NAME = configs['cluster_name']
self.AUTOSCALING_GROUP_NAME = configs['autoscaling_group_name']
self.LAUNCH_CONFIGURATION_NAME = configs['launch_configuration_name']
self.AUTOSCALE_UP_ALARMS = configs['autoscale_up_alarms']
self.AUTOSCALE_DOWN_ALARMS = configs['autoscale_down_alarms']
self.ECS_TASK_CLEANUP_WAIT = configs['ecs_task_cleanup_wait_duration']
self.template = Template()
self.user_data = UserData(configs_directory)
self.init_template()
class SitHelperTest(unittest.TestCase):
def setUp(self):
self.sit_helper = SITHelper('tests/helpers/configs')
self.sit_helper_empty = SITHelper('tests/helpers/empty_configs')
def test_get_custom_user_data(self):
self.assertEquals(self.sit_helper.get_custom_user_data(), "'test'\n")
def test_get_roles(self):
self.assertEquals(self.sit_helper.get_roles(), ['php', 'lb'])
self.assertEquals(self.sit_helper.get_roles(), ['php', 'lb'])
self.assertEquals(self.sit_helper_empty.get_roles(), False)
def test_get_states_for_role(self):
self.assertEquals(self.sit_helper.get_states_for_role('php'), ['apache', 'server'])
self.assertEquals(self.sit_helper.get_states_for_role('lb'), ['lb', 'server'])
self.assertEquals(self.sit_helper.get_states_for_role('saltmaster'),
"Failed to find state list for role: saltmaster. error: 'saltmaster'")
def test_get_configs(self):
self.assertEquals(self.sit_helper.get_configs('roles'), {'php': ['apache', 'server'], 'lb': ['lb', 'server']})
def setUp(self):
self.sit_helper = SITHelper('tests/helpers/configs')
self.sit_helper_empty = SITHelper('tests/helpers/empty_configs')
def setUp(self):
self.sit_helper = SITHelper('tests/helpers/configs')
self.sit_helper_empty = SITHelper('tests/helpers/empty_configs')
def __init__(self, configs_directory='tests/sit/configs'):
sit_helper = SITHelper(configs_directory)
CONFIGS = sit_helper.get_configs('troposphere')
self.LAUNCH_CONFIGURATION_NAME = CONFIGS['launch_configuration_name']
self.AUTOSCALING_GROUP_NAME = CONFIGS['autoscaling_group_name']
self.CUSTOM_USER_DATA = sit_helper.get_custom_user_data()
def __init__(self, configs_directory='configs', session=None):
if session is None:
sit_configs = SITHelper(configs_directory).get_configs('sit')
session = Session(profile_name=sit_configs['profile_name'])
self.ec2_client = session.client('ec2')
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
def __init__(self, configs_directory=None, session=None):
if session is None:
sit_configs = SITHelper(configs_directory).get_configs('sit')
session = Session(profile_name=sit_configs['profile_name'])
self.cf_client = session.client('cloudformation')
def __init__(self, configs_directory="tests/sit/configs"):
sit_helper = SITHelper(configs_directory)
CONFIGS = sit_helper.get_configs("troposphere")
self.LAUNCH_CONFIGURATION_NAME = CONFIGS["launch_configuration_name"]
self.AUTOSCALING_GROUP_NAME = CONFIGS["autoscaling_group_name"]
self.CUSTOM_USER_DATA = sit_helper.get_custom_user_data()
