def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
def __init__(self,
job_name=None,
build_number=None,
master_ip=None,
configs_directory=None,
session=None):
self.check_sit(configs_directory=configs_directory, session=session)
sit_helper = SITHelper(configs_directory)
self.configs_directory = configs_directory
sit_configs = sit_helper.get_configs('sit')
troposphere_configs = sit_helper.get_configs('troposphere')
self.PROFILE = sit_configs['profile_name']
self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs[
'autoscaling_group_name']
self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name']
self.STACK_NAME = troposphere_configs['stack_name']
self.ROLES = sit_helper.get_roles()
self.ATTEMPT_LIMIT = sit_configs['attempt_limit']
self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit']
self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size']
self.SAVE_LOGS = sit_configs['save_logs']
self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir']
self.REDIS_HOST = sit_configs.get('redis_host', None)
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
self.family = self.join_items([job_name, build_number])
self.master_ip = master_ip
self.is_build_successful = True
self.instance_was_launched = False
self.instance_was_terminated = False
self.init_boto_clients(session=session)
self.cluster = self.get_cluster()
self.autoscaling_group = self.get_autoscaling_group()
self.running_task_ids = []
def setUp(self):
session = Session(region_name='us-west-1')
current_dir = os.path.dirname(os.path.realpath(__file__))
test_dir = '{0}/test_data'.format(current_dir)
pill = placebo.attach(session, test_dir)
pill.playback()
self.configs_directory = 'tests/sit/configs'
self.cf_helper_placebo = CFHelper(
configs_directory=self.configs_directory, session=session)
self.sit_loader_placebo = SITLoader(
configs_directory=self.configs_directory, session=session)
def __init__(self, configs_directory='configs', session=None):
troposphere_configs = SITHelper(configs_directory).get_configs(
'troposphere')
self.STACK_NAME = troposphere_configs['stack_name']
self.TAG_VALUE = troposphere_configs['tag_value']
self.AMI_ID = troposphere_configs['ami_id']
self.SUBNET = troposphere_configs['subnet']
self.SECURITY_GROUPS = troposphere_configs['security_groups']
self.KEY_NAME = troposphere_configs['key_name']
self.AMI_URL = troposphere_configs['ami_url']
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
self.sit_template = SITTemplate(configs_directory)
self.template = self.sit_template.template
self.template_json = self.template.to_json()
def __init__(self, job_name=None, build_number=None, master_ip=None, configs_directory=None, session=None):
self.check_sit(configs_directory=configs_directory, session=session)
sit_helper = SITHelper(configs_directory)
self.configs_directory = configs_directory
sit_configs = sit_helper.get_configs('sit')
troposphere_configs = sit_helper.get_configs('troposphere')
self.PROFILE = sit_configs['profile_name']
self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs['autoscaling_group_name']
self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name']
self.STACK_NAME = troposphere_configs['stack_name']
self.ROLES = sit_helper.get_roles()
self.ATTEMPT_LIMIT = sit_configs['attempt_limit']
self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit']
self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size']
self.SAVE_LOGS = sit_configs['save_logs']
self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir']
self.REDIS_HOST = sit_configs.get('redis_host', None)
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
self.family = self.join_items([job_name, build_number])
self.master_ip = master_ip
self.is_build_successful = True
self.instance_was_launched = False
self.instance_was_terminated = False
self.init_boto_clients(session=session)
self.cluster = self.get_cluster()
self.autoscaling_group = self.get_autoscaling_group()
self.running_task_ids = []
class CheckSIT(object):
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
def check_configs_are_set(self):
missing_configs = [config for config, value in self.SIT_CONFIGS.iteritems() if value is None]
if missing_configs:
Log.error('The following configs are not set: {0}'.format(missing_configs))
def check_roles_file(self):
if self.ROLES < 1:
Log.error('roles.yml file is not setup properly. You require at least one role to test')
def check_roles_not_empty(self):
empty_roles = [role for role in self.ROLES if not self.sit_helper.get_states_for_role(role)]
if empty_roles:
Log.error('The following servers are missing roles inside of roles.yml file: {0}'.format(empty_roles))
def check_stack_exists(self):
stack_name = self.TROPOSPHERE_CONFIGS['stack_name']
if not self.cf_helper.get_stack_info(stack_name):
Log.error('Stack "{0}" does not exist. Please run setup_troposphere'.format(stack_name))
def run(self):
self.check_configs_are_set()
self.check_roles_file()
self.check_roles_not_empty()
self.check_stack_exists()
def setUp(self):
session = Session(region_name='us-west-1')
current_dir = os.path.dirname(os.path.realpath(__file__))
test_dir = '{0}/test_data'.format(current_dir)
pill = placebo.attach(session, test_dir)
pill.playback()
self.configs_directory = 'tests/sit/configs'
self.cf_helper_placebo = CFHelper(configs_directory=self.configs_directory, session=session)
self.sit_loader_placebo = SITLoader(configs_directory=self.configs_directory, session=session)
def mock_cf_helper_true_responses(self):
cf_helper = CFHelper(configs_directory=self.configs_directory,
session=self.session)
cf_helper.get_stack_info = MagicMock(return_value=False)
cf_helper.validate_template = MagicMock(return_value=True)
cf_helper.create_or_update_stack = MagicMock(return_value=True)
cf_helper.create_stack = MagicMock(return_value=True)
cf_helper.stack_was_created_successfully = MagicMock(return_value=True)
self.sit_loader.cf_helper = cf_helper
def __init__(self, configs_directory='configs', session=None):
troposphere_configs = SITHelper(configs_directory).get_configs('troposphere')
self.STACK_NAME = troposphere_configs['stack_name']
self.TAG_VALUE = troposphere_configs['tag_value']
self.AMI_ID = troposphere_configs['ami_id']
self.SUBNET = troposphere_configs['subnet']
self.SECURITY_GROUPS = troposphere_configs['security_groups']
self.KEY_NAME = troposphere_configs['key_name']
self.AMI_URL = troposphere_configs['ami_url']
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
self.sit_template = SITTemplate(configs_directory)
self.template = self.sit_template.template
self.template_json = self.template.to_json()
def mock_cf_helper_true_responses(self):
cf_helper = CFHelper(configs_directory=self.configs_directory, session=self.session)
cf_helper.get_stack_info = MagicMock(return_value=False)
cf_helper.validate_template = MagicMock(return_value=True)
cf_helper.create_or_update_stack = MagicMock(return_value=True)
cf_helper.create_stack = MagicMock(return_value=True)
cf_helper.stack_was_created_successfully = MagicMock(return_value=True)
self.sit_loader.cf_helper = cf_helper
class CheckSIT(object):
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
def check_configs_are_set(self):
missing_configs = [
config for config, value in self.SIT_CONFIGS.iteritems()
if value is None
]
if missing_configs:
Log.error('The following configs are not set: {0}'.format(
missing_configs))
def check_roles_file(self):
if self.ROLES < 1:
Log.error(
'roles.yml file is not setup properly. You require at least one role to test'
)
def check_roles_not_empty(self):
empty_roles = [
role for role in self.ROLES
if not self.sit_helper.get_states_for_role(role)
]
if empty_roles:
Log.error(
'The following servers are missing roles inside of roles.yml file: {0}'
.format(empty_roles))
def check_stack_exists(self):
stack_name = self.TROPOSPHERE_CONFIGS['stack_name']
if not self.cf_helper.get_stack_info(stack_name):
Log.error(
'Stack "{0}" does not exist. Please run setup_troposphere'.
format(stack_name))
def run(self):
self.check_configs_are_set()
self.check_roles_file()
self.check_roles_not_empty()
self.check_stack_exists()
class SITLoader(object):
def __init__(self, configs_directory='configs', session=None):
troposphere_configs = SITHelper(configs_directory).get_configs(
'troposphere')
self.STACK_NAME = troposphere_configs['stack_name']
self.TAG_VALUE = troposphere_configs['tag_value']
self.AMI_ID = troposphere_configs['ami_id']
self.SUBNET = troposphere_configs['subnet']
self.SECURITY_GROUPS = troposphere_configs['security_groups']
self.KEY_NAME = troposphere_configs['key_name']
self.AMI_URL = troposphere_configs['ami_url']
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
self.sit_template = SITTemplate(configs_directory)
self.template = self.sit_template.template
self.template_json = self.template.to_json()
def validate_template(self):
self.cf_helper.validate_template(self.template_json)
def create_or_update_stack(self):
if self.cf_helper.stack_exists(self.STACK_NAME):
self.cf_helper.update_stack(self.STACK_NAME, self.template_json,
self.TAG_VALUE)
else:
self.cf_helper.create_stack(self.STACK_NAME, self.template_json,
self.TAG_VALUE)
def stack_created_successfully(self):
if not self.cf_helper.stack_was_created_successfully(self.STACK_NAME):
Log.error('stack was not created')
def run(self):
start = time()
self.validate_template()
self.create_or_update_stack()
self.stack_created_successfully()
end = time()
logging.info('Your stack is up and ready to go!')
logging.info('Elapsed time: {0} seconds'.format(end - start))
class SITLoader(object):
def __init__(self, configs_directory='configs', session=None):
troposphere_configs = SITHelper(configs_directory).get_configs('troposphere')
self.STACK_NAME = troposphere_configs['stack_name']
self.TAG_VALUE = troposphere_configs['tag_value']
self.AMI_ID = troposphere_configs['ami_id']
self.SUBNET = troposphere_configs['subnet']
self.SECURITY_GROUPS = troposphere_configs['security_groups']
self.KEY_NAME = troposphere_configs['key_name']
self.AMI_URL = troposphere_configs['ami_url']
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
self.sit_template = SITTemplate(configs_directory)
self.template = self.sit_template.template
self.template_json = self.template.to_json()
def validate_template(self):
self.cf_helper.validate_template(self.template_json)
def create_or_update_stack(self):
if self.cf_helper.stack_exists(self.STACK_NAME):
self.cf_helper.update_stack(self.STACK_NAME, self.template_json, self.TAG_VALUE)
else:
self.cf_helper.create_stack(self.STACK_NAME, self.template_json, self.TAG_VALUE)
def stack_created_successfully(self):
if not self.cf_helper.stack_was_created_successfully(self.STACK_NAME):
Log.error('stack was not created')
def run(self):
start = time()
self.validate_template()
self.create_or_update_stack()
self.stack_created_successfully()
end = time()
logging.info('Your stack is up and ready to go!')
logging.info('Elapsed time: {0} seconds'.format(end - start))
class CFHelperTest(unittest.TestCase):
def setUp(self):
session = Session(region_name='us-west-1')
current_dir = os.path.dirname(os.path.realpath(__file__))
test_dir = '{0}/test_data'.format(current_dir)
pill = placebo.attach(session, test_dir)
pill.playback()
self.configs_directory = 'tests/sit/configs'
self.cf_helper_placebo = CFHelper(
configs_directory=self.configs_directory, session=session)
self.sit_loader_placebo = SITLoader(
configs_directory=self.configs_directory, session=session)
@raises(SystemExit)
def test_validate_template(self):
template_body = SITTemplate(self.configs_directory).template.to_json()
negative_result = self.cf_helper_placebo.validate_template(
template_body)
self.assertEquals(negative_result, None)
def test_create_stack_success(self):
positive_result = self.cf_helper_placebo.create_stack(
self.sit_loader_placebo.STACK_NAME,
self.sit_loader_placebo.template_json,
self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
def test_update_stack_success(self):
positive_result = self.cf_helper_placebo.update_stack(
self.sit_loader_placebo.STACK_NAME,
self.sit_loader_placebo.template_json,
self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
@raises(SystemExit)
def test_create_stack_failure(self):
positive_result = self.cf_helper_placebo.create_stack(
self.sit_loader_placebo.STACK_NAME,
self.sit_loader_placebo.template_json,
self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
self.cf_helper_placebo.create_stack(
self.sit_loader_placebo.STACK_NAME,
self.sit_loader_placebo.template_json,
self.sit_loader_placebo.TAG_VALUE)
@raises(SystemExit)
def test_update_stack_failure(self):
positive_result = self.cf_helper_placebo.update_stack(
self.sit_loader_placebo.STACK_NAME,
self.sit_loader_placebo.template_json,
self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
self.cf_helper_placebo.update_stack(
self.sit_loader_placebo.STACK_NAME,
self.sit_loader_placebo.template_json,
self.sit_loader_placebo.TAG_VALUE)
def test_stack_was_created_successfully(self):
attempts = 25 # set attempts to the max so that it only tries once
positive_result = self.cf_helper_placebo.stack_was_created_successfully(
self.sit_loader_placebo.STACK_NAME, attempts, 0)
self.assertEquals(positive_result, True)
# failed status
negative_result = self.cf_helper_placebo.stack_was_created_successfully(
self.sit_loader_placebo.STACK_NAME, attempts, 0)
self.assertEquals(negative_result, False)
# too many attempts
negative_result = self.cf_helper_placebo.stack_was_created_successfully(
self.sit_loader_placebo.STACK_NAME, attempts + 1, 0)
self.assertEquals(negative_result, False)
# bad status
negative_result = self.cf_helper_placebo.stack_was_created_successfully(
self.sit_loader_placebo.STACK_NAME, attempts, 0)
self.assertEquals(negative_result, False)
@raises(SystemExit)
def test_get_resource_name(self):
# this call will raise the SystemExit because the stack has already been created
self.cf_helper_placebo.get_resource_name(
self.sit_loader_placebo.STACK_NAME, 'AutoFailingGroup')
class ReviewJob(object):
BOTO_CLIENT_TYPE_ECS = 'ecs'
BOTO_CLIENT_TYPE_EC2 = 'ec2'
BOTO_CLIENT_TYPE_AUTOSCALING = 'autoscaling'
TASK_FAILURE_REASON = 'RESOURCE'
CONTAINER_INSTANCE_WAIT = 30
CLUSTER_RESOURCE_WAIT = 60
TASK_COMPLETION_WAIT = 30
def __init__(self,
job_name=None,
build_number=None,
master_ip=None,
configs_directory=None,
session=None):
self.check_sit(configs_directory=configs_directory, session=session)
sit_helper = SITHelper(configs_directory)
self.configs_directory = configs_directory
sit_configs = sit_helper.get_configs('sit')
troposphere_configs = sit_helper.get_configs('troposphere')
self.PROFILE = sit_configs['profile_name']
self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs[
'autoscaling_group_name']
self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name']
self.STACK_NAME = troposphere_configs['stack_name']
self.ROLES = sit_helper.get_roles()
self.ATTEMPT_LIMIT = sit_configs['attempt_limit']
self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit']
self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size']
self.SAVE_LOGS = sit_configs['save_logs']
self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir']
self.REDIS_HOST = sit_configs.get('redis_host', None)
self.cf_helper = CFHelper(configs_directory=configs_directory,
session=session)
self.family = self.join_items([job_name, build_number])
self.master_ip = master_ip
self.is_build_successful = True
self.instance_was_launched = False
self.instance_was_terminated = False
self.init_boto_clients(session=session)
self.cluster = self.get_cluster()
self.autoscaling_group = self.get_autoscaling_group()
self.running_task_ids = []
def check_sit(self, configs_directory, session):
CheckSIT(configs_directory=configs_directory, session=session).run()
def init_boto_clients(self, session=None):
if session is None:
session = Session(profile_name=self.PROFILE)
self.ecs_client = self.get_boto_client(session,
self.BOTO_CLIENT_TYPE_ECS)
self.autoscaling_client = self.get_boto_client(
session, self.BOTO_CLIENT_TYPE_AUTOSCALING)
self.ec2_client = self.get_boto_client(session,
self.BOTO_CLIENT_TYPE_EC2)
def run(self):
self.prepare_cluster()
tasks = self.register_tasks()
self.start_tasks(tasks)
self.wait_for_tasks_to_complete()
def prepare_cluster(self):
capacity = self.get_current_desired_capacity()
logging.info('Current SIT Cluster capacity: {0}'.format(capacity))
if capacity < self.INITIAL_CLUSTER_SIZE:
logging.info(
'Cluster is currently empty or inadequate. Setting Desired Capacity to {0}'
.format(self.INITIAL_CLUSTER_SIZE))
self.set_current_desired_capacity(self.INITIAL_CLUSTER_SIZE)
self.wait_for_first_instance()
def scale_up_cluster(self):
capacity = self.get_current_desired_capacity()
new_cluster_size = capacity + self.INITIAL_CLUSTER_SIZE
logging.info(
'Increasing Current SIT Cluster capacity from {0} to {1}'.format(
capacity, new_cluster_size))
self.set_current_desired_capacity(new_cluster_size)
def wait_for_first_instance(self, attempt=1):
if attempt > 10:
self.error(
'Timed out waiting for instance to become available in sit cluster.'
)
cluster_instances = self.ecs_client.list_container_instances(
cluster=self.cluster)['containerInstanceArns']
if not cluster_instances:
logging.info(
'First instance not in cluster yet. Waiting for {0} seconds...'
.format(self.CONTAINER_INSTANCE_WAIT))
sleep(self.CONTAINER_INSTANCE_WAIT)
self.wait_for_first_instance(attempt + 1)
else:
self.display_instances_details(cluster_instances)
def display_instances_details(self, cluster_instances):
try:
cluster_container_ids = map(lambda arn: arn.split('/')[1],
cluster_instances)
cluster_instance_details = self.ecs_client.describe_container_instances(
containerInstances=cluster_container_ids,
cluster=self.cluster)['containerInstances']
cluster_instance_ids = map(lambda x: x['ec2InstanceId'],
cluster_instance_details)
logging.info(
'SIT Cluster Instance Ids: {0}'.format(cluster_instance_ids))
self.display_instances_private_ips(cluster_instance_ids)
except Exception as e:
logging.warn(
'Failed to display instance private ip. Error: {0}'.format(e))
def display_instances_private_ips(self, instance_ids):
try:
instance_reservations = self.ec2_client.describe_instances(
InstanceIds=instance_ids)['Reservations']
ips = [
instance['PrivateIpAddress']
for reservation in instance_reservations
for instance in reservation['Instances']
]
logging.info('SIT Cluster Instance IPs: {0}'.format(ips))
except Exception as e:
logging.warn(
'Failed to describe instance(s): {0}. Error: {1}'.format(
instance_ids, e))
def get_autoscaling_group(self):
try:
return self.cf_helper.get_resource_name(
self.STACK_NAME, self.LOGICAL_AUTOSCALING_GROUP_NAME)
except Exception as e:
self.error('Unable to get the Autoscaling Group name', e)
def register_tasks(self):
tasks = []
for role in self.ROLES:
family = self.join_items([self.family, role])
self.register_task(family, role)
tasks.append(family)
return tasks
def register_task(self, family, role):
try:
self.ecs_client.register_task_definition(
family=family,
containerDefinitions=[
self.get_container_definitions(role, family)
])
except Exception as e:
self.error(
'Failed to register tasks for family: {0}, role: {1}'.format(
family, role), e)
def get_container_definitions(self, role, family):
container = Container(configs_directory=self.configs_directory,
env='local',
role=role,
family=family,
master_ip=self.master_ip,
redis_host=self.REDIS_HOST)
return container.get_container_definitions()
def start_tasks(self, tasks):
try:
for task in tasks:
response = self.attempt_start_task(task)
logging.info('Task: {0} has started running..'.format(task))
self.running_task_ids.append(response['tasks'][0]['taskArn'])
except Exception as e:
self.error('Failed to run tasks', e)
def attempt_start_task(self, task, attempt=1, scaleup_required=True):
if attempt > self.RESOURCES_ATTEMPT_LIMIT:
self.error(
'Task {0} could not start. Timed out waiting for resources.'.
format(task))
response = self.ecs_client.run_task(cluster=self.cluster,
taskDefinition=task)
if response['failures'] and self.TASK_FAILURE_REASON in response[
'failures'][0]['reason']:
logging.info(
'Task: {0} failed to start due to unavailable Resources. Waiting 60 seconds.'
.format(task))
if scaleup_required:
self.scale_up_cluster()
sleep(self.CLUSTER_RESOURCE_WAIT)
response = self.attempt_start_task(task,
attempt + 1,
scaleup_required=False)
elif response['failures']:
raise ValueError(
'Task: {0} failed to start due to unexpected reason:{1}'.
format(task, response['failures'][0]['reason']))
return response
def wait_for_tasks_to_complete(self, attempt=0):
if attempt > self.ATTEMPT_LIMIT:
self.error(
'{0} attempts have gone by. Initiating termination sequence'.
format(self.ATTEMPT_LIMIT))
task_responses = self.get_task_responses()
try:
running_tasks = filter(
lambda task: task['lastStatus'] != "STOPPED", task_responses)
self.running_task_ids = map(lambda task: task['taskArn'],
running_tasks)
if running_tasks:
logging.info(
'{0} Tasks are still running. Waiting for {1} seconds'.
format(len(self.running_task_ids),
self.TASK_COMPLETION_WAIT))
sleep(self.TASK_COMPLETION_WAIT)
return self.wait_for_tasks_to_complete(attempt + 1)
except Exception as e:
self.error('Failed determining status of task', e)
logging.info("Tasks are complete")
def get_task_responses(self):
try:
return self.ecs_client.describe_tasks(
cluster=self.cluster, tasks=self.running_task_ids)['tasks']
except Exception as e:
self.error('Failed to retrieve tasks from cluster.', e)
def check_and_print_results(self):
redis_client = RedisClient(self.REDIS_HOST)
for role in self.ROLES:
family = self.join_items([self.family, role])
highstate_result = redis_client.get_highstate_result(family)
try:
logging.info('printing results for server: {0}'.format(role))
parsed_result = json.loads(highstate_result)
return_results = parsed_result.pop('return')
json_results = json.dumps(parsed_result, indent=4)
yaml_dump = yaml.safe_dump(return_results)
logging.info('{0}\n{1}\n'.format(yaml_dump, json_results))
if self.SAVE_LOGS:
self.check_for_log_dir()
self.write_to_log_file(yaml_dump, role)
self.write_to_log_file(json_results, role)
except Exception as e:
logging.info('Result:{0} Exception:{1}'.format(
highstate_result, e))
try:
if self.highstate_failed(highstate_result):
self.is_build_successful = False
except:
self.is_build_successful = False
def highstate_failed(self, result):
try:
possible_failures = [
'"result": false', 'Data failed to compile:',
'Pillar failed to render with the following messages:',
'Detected conflicting IDs', 'Cannot extend ID',
'not available on the salt master or through a configured fileserver'
]
failures = [failure in result for failure in possible_failures]
if True not in failures:
failures = self.check_regex_failure(failures, result)
return True in failures
except:
logging.info('Error finding if there was a failure in the result')
return True
def check_regex_failure(self, failures, result):
regex_failure = r"Rendering SLS '.*' failed:"
failures.append(bool(re.search(regex_failure, result)))
return failures
def check_for_log_dir(self):
if not os.path.exists(self.HIGHSTATE_LOG_DIR):
os.makedirs(self.HIGHSTATE_LOG_DIR)
def write_to_log_file(self, result, role):
try:
logging.info("Writing results to logs")
with open('{0}/{1}.txt'.format(self.HIGHSTATE_LOG_DIR, role),
'a') as log_file:
log_file.write(result)
except Exception as e:
self.error('Failed to write to logs', e)
def fail_build_if_failures_exist(self):
if not self.is_build_successful:
self.error('build is not successful')
def get_cluster(self):
try:
return self.cf_helper.get_resource_name(self.STACK_NAME,
self.LOGICAL_CLUSTER_NAME)
except Exception as e:
self.error('Failed to retrieve the cluster', e)
def set_current_desired_capacity(self, capacity=None):
try:
return self.autoscaling_client.set_desired_capacity(
AutoScalingGroupName=self.autoscaling_group,
DesiredCapacity=capacity)
except Exception as e:
self.error(
'Failed to increase the desired capacity for Autoscaling Group: {0}'
.format(self.autoscaling_group), e)
def get_current_desired_capacity(self):
try:
return self.autoscaling_client.describe_auto_scaling_groups(
AutoScalingGroupNames=[self.autoscaling_group]
)['AutoScalingGroups'][0]['DesiredCapacity']
except Exception as e:
self.error(
'Failed to retrieve the desired capacity for Austoscaling Group: {0}.'
.format(self.autoscaling_group), e)
def get_boto_client(self, session, client_name):
try:
return session.client(client_name)
except Exception as e:
self.error(
'Failed to get boto client for: {0}'.format(client_name), e)
def join_items(self, items=list(), delimeter='-'):
try:
return delimeter.join(items)
except Exception as e:
self.error(
'unable to join items: {0} with delimeter: {1}.'.format(
items, delimeter), e)
def terminate_running_tasks(self):
task_definitions = []
try:
for task_id in self.running_task_ids:
response = self.ecs_client.stop_task(task=task_id,
cluster=self.cluster)
task_definitions.append(response['task']['taskDefinitionArn'])
except Exception as e:
logging.error('Error occurred while terminating tasks.', e)
finally:
return task_definitions
def error(self, message, e=None, error_code=2):
terminated_tasks = self.terminate_running_tasks()
logging.info(
'{0}. Exception: {1}\nFollowing tasks have been terminated: {2}'.
format(message, e, terminated_tasks))
exit(error_code)
class CFHelperTest(unittest.TestCase):
def setUp(self):
session = Session(region_name='us-west-1')
current_dir = os.path.dirname(os.path.realpath(__file__))
test_dir = '{0}/test_data'.format(current_dir)
pill = placebo.attach(session, test_dir)
pill.playback()
self.configs_directory = 'tests/sit/configs'
self.cf_helper_placebo = CFHelper(configs_directory=self.configs_directory, session=session)
self.sit_loader_placebo = SITLoader(configs_directory=self.configs_directory, session=session)
@raises(SystemExit)
def test_validate_template(self):
template_body = SITTemplate(self.configs_directory).template.to_json()
negative_result = self.cf_helper_placebo.validate_template(template_body)
self.assertEquals(negative_result, None)
def test_create_stack_success(self):
positive_result = self.cf_helper_placebo.create_stack(self.sit_loader_placebo.STACK_NAME, self.sit_loader_placebo.template_json, self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
def test_update_stack_success(self):
positive_result = self.cf_helper_placebo.update_stack(self.sit_loader_placebo.STACK_NAME, self.sit_loader_placebo.template_json, self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
@raises(SystemExit)
def test_create_stack_failure(self):
positive_result = self.cf_helper_placebo.create_stack(self.sit_loader_placebo.STACK_NAME, self.sit_loader_placebo.template_json, self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
self.cf_helper_placebo.create_stack(self.sit_loader_placebo.STACK_NAME, self.sit_loader_placebo.template_json, self.sit_loader_placebo.TAG_VALUE)
@raises(SystemExit)
def test_update_stack_failure(self):
positive_result = self.cf_helper_placebo.update_stack(self.sit_loader_placebo.STACK_NAME, self.sit_loader_placebo.template_json, self.sit_loader_placebo.TAG_VALUE)
self.assertEquals(positive_result, None)
self.cf_helper_placebo.update_stack(self.sit_loader_placebo.STACK_NAME, self.sit_loader_placebo.template_json, self.sit_loader_placebo.TAG_VALUE)
def test_stack_was_created_successfully(self):
attempts = 25 # set attempts to the max so that it only tries once
positive_result = self.cf_helper_placebo.stack_was_created_successfully(self.sit_loader_placebo.STACK_NAME, attempts, 0)
self.assertEquals(positive_result, True)
# failed status
negative_result = self.cf_helper_placebo.stack_was_created_successfully(self.sit_loader_placebo.STACK_NAME, attempts, 0)
self.assertEquals(negative_result, False)
# too many attempts
negative_result = self.cf_helper_placebo.stack_was_created_successfully(self.sit_loader_placebo.STACK_NAME, attempts + 1, 0)
self.assertEquals(negative_result, False)
# bad status
negative_result = self.cf_helper_placebo.stack_was_created_successfully(self.sit_loader_placebo.STACK_NAME, attempts, 0)
self.assertEquals(negative_result, False)
@raises(SystemExit)
def test_get_resource_name(self):
# this call will raise the SystemExit because the stack has already been created
self.cf_helper_placebo.get_resource_name(self.sit_loader_placebo.STACK_NAME, 'AutoFailingGroup')
class ReviewJob(object):
BOTO_CLIENT_TYPE_ECS = 'ecs'
BOTO_CLIENT_TYPE_EC2 = 'ec2'
BOTO_CLIENT_TYPE_AUTOSCALING = 'autoscaling'
TASK_FAILURE_REASON = 'RESOURCE'
CONTAINER_INSTANCE_WAIT = 30
CLUSTER_RESOURCE_WAIT = 60
TASK_COMPLETION_WAIT = 30
def __init__(self, job_name=None, build_number=None, master_ip=None, configs_directory=None, session=None):
self.check_sit(configs_directory=configs_directory, session=session)
sit_helper = SITHelper(configs_directory)
self.configs_directory = configs_directory
sit_configs = sit_helper.get_configs('sit')
troposphere_configs = sit_helper.get_configs('troposphere')
self.PROFILE = sit_configs['profile_name']
self.LOGICAL_AUTOSCALING_GROUP_NAME = troposphere_configs['autoscaling_group_name']
self.LOGICAL_CLUSTER_NAME = troposphere_configs['cluster_name']
self.STACK_NAME = troposphere_configs['stack_name']
self.ROLES = sit_helper.get_roles()
self.ATTEMPT_LIMIT = sit_configs['attempt_limit']
self.RESOURCES_ATTEMPT_LIMIT = sit_configs['resources_attempt_limit']
self.INITIAL_CLUSTER_SIZE = sit_configs['initial_cluster_size']
self.SAVE_LOGS = sit_configs['save_logs']
self.HIGHSTATE_LOG_DIR = sit_configs['highstate_log_dir']
self.REDIS_HOST = sit_configs.get('redis_host', None)
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
self.family = self.join_items([job_name, build_number])
self.master_ip = master_ip
self.is_build_successful = True
self.instance_was_launched = False
self.instance_was_terminated = False
self.init_boto_clients(session=session)
self.cluster = self.get_cluster()
self.autoscaling_group = self.get_autoscaling_group()
self.running_task_ids = []
def check_sit(self, configs_directory, session):
CheckSIT(configs_directory=configs_directory, session=session).run()
def init_boto_clients(self, session=None):
if session is None:
session = Session(profile_name=self.PROFILE)
self.ecs_client = self.get_boto_client(session, self.BOTO_CLIENT_TYPE_ECS)
self.autoscaling_client = self.get_boto_client(session, self.BOTO_CLIENT_TYPE_AUTOSCALING)
self.ec2_client = self.get_boto_client(session, self.BOTO_CLIENT_TYPE_EC2)
def run(self):
self.prepare_cluster()
tasks = self.register_tasks()
self.start_tasks(tasks)
self.wait_for_tasks_to_complete()
def prepare_cluster(self):
capacity = self.get_current_desired_capacity()
logging.info('Current SIT Cluster capacity: {0}'.format(capacity))
if capacity < self.INITIAL_CLUSTER_SIZE:
logging.info('Cluster is currently empty or inadequate. Setting Desired Capacity to {0}'.format(
self.INITIAL_CLUSTER_SIZE))
self.set_current_desired_capacity(self.INITIAL_CLUSTER_SIZE)
self.wait_for_first_instance()
def scale_up_cluster(self):
capacity = self.get_current_desired_capacity()
new_cluster_size = capacity + self.INITIAL_CLUSTER_SIZE
logging.info('Increasing Current SIT Cluster capacity from {0} to {1}'.format(capacity, new_cluster_size))
self.set_current_desired_capacity(new_cluster_size)
def wait_for_first_instance(self, attempt=1):
if attempt > 10:
self.error('Timed out waiting for instance to become available in sit cluster.')
cluster_instances = self.ecs_client.list_container_instances(cluster=self.cluster)['containerInstanceArns']
if not cluster_instances:
logging.info('First instance not in cluster yet. Waiting for {0} seconds...'.format(self.CONTAINER_INSTANCE_WAIT))
sleep(self.CONTAINER_INSTANCE_WAIT)
self.wait_for_first_instance(attempt+1)
else:
self.display_instances_details(cluster_instances)
def display_instances_details(self, cluster_instances):
try:
cluster_container_ids = map(lambda arn: arn.split('/')[1], cluster_instances)
cluster_instance_details = self.ecs_client.describe_container_instances(
containerInstances=cluster_container_ids, cluster=self.cluster)['containerInstances']
cluster_instance_ids = map(lambda x: x['ec2InstanceId'], cluster_instance_details)
logging.info('SIT Cluster Instance Ids: {0}'.format(cluster_instance_ids))
self.display_instances_private_ips(cluster_instance_ids)
except Exception as e:
logging.warn('Failed to display instance private ip. Error: {0}'.format(e))
def display_instances_private_ips(self, instance_ids):
try:
instance_reservations = self.ec2_client.describe_instances(InstanceIds=instance_ids)['Reservations']
ips = [instance['PrivateIpAddress'] for reservation in instance_reservations for instance in reservation['Instances']]
logging.info('SIT Cluster Instance IPs: {0}'.format(ips))
except Exception as e:
logging.warn('Failed to describe instance(s): {0}. Error: {1}'.format(instance_ids, e))
def get_autoscaling_group(self):
try:
return self.cf_helper.get_resource_name(self.STACK_NAME, self.LOGICAL_AUTOSCALING_GROUP_NAME)
except Exception as e:
self.error('Unable to get the Autoscaling Group name', e)
def register_tasks(self):
tasks = []
for role in self.ROLES:
family = self.join_items([self.family, role])
self.register_task(family, role)
tasks.append(family)
return tasks
def register_task(self, family, role):
try:
self.ecs_client.register_task_definition(
family=family,
containerDefinitions=[self.get_container_definitions(role, family)]
)
except Exception as e:
self.error('Failed to register tasks for family: {0}, role: {1}'.format(family, role), e)
def get_container_definitions(self, role, family):
container = Container(configs_directory=self.configs_directory, env='local', role=role, family=family, master_ip=self.master_ip, redis_host=self.REDIS_HOST)
return container.get_container_definitions()
def start_tasks(self, tasks):
try:
for task in tasks:
response = self.attempt_start_task(task)
logging.info('Task: {0} has started running..'.format(task))
self.running_task_ids.append(response['tasks'][0]['taskArn'])
except Exception as e:
self.error('Failed to run tasks', e)
def attempt_start_task(self, task, attempt=1, scaleup_required=True):
if attempt > self.RESOURCES_ATTEMPT_LIMIT:
self.error('Task {0} could not start. Timed out waiting for resources.'.format(task))
response = self.ecs_client.run_task(
cluster=self.cluster,
taskDefinition=task
)
if response['failures'] and self.TASK_FAILURE_REASON in response['failures'][0]['reason']:
logging.info('Task: {0} failed to start due to unavailable Resources. Waiting 60 seconds.'.format(task))
if scaleup_required:
self.scale_up_cluster()
sleep(self.CLUSTER_RESOURCE_WAIT)
response = self.attempt_start_task(task, attempt+1, scaleup_required=False)
elif response['failures']:
raise ValueError('Task: {0} failed to start due to unexpected reason:{1}'.format(
task, response['failures'][0]['reason']))
return response
def wait_for_tasks_to_complete(self, attempt=0):
if attempt > self.ATTEMPT_LIMIT:
self.error('{0} attempts have gone by. Initiating termination sequence'.format(self.ATTEMPT_LIMIT))
task_responses = self.get_task_responses()
try:
running_tasks = filter(lambda task: task['lastStatus'] != "STOPPED", task_responses)
self.running_task_ids = map(lambda task: task['taskArn'], running_tasks)
if running_tasks:
logging.info('{0} Tasks are still running. Waiting for {1} seconds'.format(
len(self.running_task_ids), self.TASK_COMPLETION_WAIT))
sleep(self.TASK_COMPLETION_WAIT)
return self.wait_for_tasks_to_complete(attempt + 1)
except Exception as e:
self.error('Failed determining status of task', e)
logging.info("Tasks are complete")
def get_task_responses(self):
try:
return self.ecs_client.describe_tasks(cluster=self.cluster, tasks=self.running_task_ids)['tasks']
except Exception as e:
self.error('Failed to retrieve tasks from cluster.', e)
def check_and_print_results(self):
redis_client = RedisClient(self.REDIS_HOST)
for role in self.ROLES:
family = self.join_items([self.family, role])
highstate_result = redis_client.get_highstate_result(family)
try:
logging.info('printing results for server: {0}'.format(role))
parsed_result = json.loads(highstate_result)
return_results = parsed_result.pop('return')
json_results = json.dumps(parsed_result, indent=4)
yaml_dump = yaml.safe_dump(return_results)
logging.info('{0}\n{1}\n'.format(yaml_dump, json_results))
if self.SAVE_LOGS:
self.check_for_log_dir()
self.write_to_log_file(yaml_dump, role)
self.write_to_log_file(json_results, role)
except Exception as e:
logging.info('Result:{0} Exception:{1}'.format(highstate_result, e))
try:
if self.highstate_failed(highstate_result):
self.is_build_successful = False
except:
self.is_build_successful = False
def highstate_failed(self, result):
try:
possible_failures = [
'"result": false',
'Data failed to compile:',
'Pillar failed to render with the following messages:',
'Detected conflicting IDs',
'Cannot extend ID'
]
failures = [failure in result for failure in possible_failures]
if True not in failures:
failures = self.check_regex_failure(failures, result)
return True in failures
except:
logging.info('Error finding if there was a failure in the result')
return True
def check_regex_failure(self, failures, result):
regex_failure = r"Rendering SLS '.*' failed:"
failures.append(bool(re.search(regex_failure, result)))
return failures
def check_for_log_dir(self):
if not os.path.exists(self.HIGHSTATE_LOG_DIR):
os.makedirs(self.HIGHSTATE_LOG_DIR)
def write_to_log_file(self, result, role):
try:
logging.info("Writing results to logs")
with open('{0}/{1}.txt'.format(self.HIGHSTATE_LOG_DIR, role), 'a') as log_file:
log_file.write(result)
except Exception as e:
self.error('Failed to write to logs', e)
def fail_build_if_failures_exist(self):
if not self.is_build_successful:
self.error('build is not successful')
def get_cluster(self):
try:
return self.cf_helper.get_resource_name(self.STACK_NAME, self.LOGICAL_CLUSTER_NAME)
except Exception as e:
self.error('Failed to retrieve the cluster', e)
def set_current_desired_capacity(self, capacity=None):
try:
return self.autoscaling_client.set_desired_capacity(
AutoScalingGroupName=self.autoscaling_group,
DesiredCapacity=capacity
)
except Exception as e:
self.error('Failed to increase the desired capacity for Autoscaling Group: {0}'.format(self.autoscaling_group), e)
def get_current_desired_capacity(self):
try:
return self.autoscaling_client.describe_auto_scaling_groups(
AutoScalingGroupNames=[self.autoscaling_group]
)['AutoScalingGroups'][0]['DesiredCapacity']
except Exception as e:
self.error('Failed to retrieve the desired capacity for Austoscaling Group: {0}.'.format(self.autoscaling_group), e)
def get_boto_client(self, session, client_name):
try:
return session.client(client_name)
except Exception as e:
self.error('Failed to get boto client for: {0}'.format(client_name), e)
def join_items(self, items=list(), delimeter='-'):
try:
return delimeter.join(items)
except Exception as e:
self.error('unable to join items: {0} with delimeter: {1}.'.format(items, delimeter), e)
def terminate_running_tasks(self):
task_definitions = []
try:
for task_id in self.running_task_ids:
response = self.ecs_client.stop_task(task=task_id, cluster=self.cluster)
task_definitions.append(response['task']['taskDefinitionArn'])
except Exception as e:
logging.error('Error occurred while terminating tasks.', e)
finally:
return task_definitions
def error(self, message, e=None, error_code=2):
terminated_tasks = self.terminate_running_tasks()
logging.info('{0}. Exception: {1}\nFollowing tasks have been terminated: {2}'.format(
message, e, terminated_tasks))
exit(error_code)
def __init__(self, configs_directory=None, session=None):
self.sit_helper = SITHelper(configs_directory)
self.ROLES = self.sit_helper.get_roles()
self.SIT_CONFIGS = self.sit_helper.get_configs('sit')
self.TROPOSPHERE_CONFIGS = self.sit_helper.get_configs('troposphere')
self.cf_helper = CFHelper(configs_directory=configs_directory, session=session)
