def scale_down(config, instance_id): """Remove one instance from the environment and put it to the pool of stopped machines""" logger = logging.getLogger(__name__) if not scaling_down_possible(config): logger.warning("Scaling down not possible, minimum number of instances is running") else: logger.info("Downscaling started.") # Deregister it from load balancer elb = aws_ec2_elb.get_load_balancer(config.get("region"), config.get("elb_name")) elb.deregister_instances([instance_id]) # stop it time.sleep(static.AUTOSCALE_DELAY_BEFORE_REMOVING_INSTANCE) # let instance respond to pending requests instance = aws_ec2.get_instance(config, instance_id) instance.stop() # Update config try: instances = config.get_list("stopped_instances") except ConfigParser.NoOptionError: instances = [] instances.append(instance_id) config.set_list("stopped_instances", instances) instances = config.get_list("instances") instances.remove(instance_id) config.set_list("instances", instances) logger.info("Downscaling completed. 1 instance stopped and is available in the pool.") if len(instances) > 1: logger.info("%d instances are currently on duty." % len(instances)) else: logger.info("1 instance is currently on duty.")
def process_instance_state(config, state): if state.instance_id in config.get_list('instances') and \ not state.state == 'InService' and \ not state.reason_code == 'Instance registration is still in progress.' and \ not state.reason_code == 'ELB': id = state.instance_id if unhealthy_instances.has_key(id): num_checks = unhealthy_instances.get(id) if num_checks + 1 == static.HEALTH_CHECKS: logger.info('Stopping unhealty instance %s for investigation' %id) unhealthy_instances.pop(id) #Instance is considered failed and should be stopped for investigation #Deregister it from load balancer elb = aws_ec2_elb.get_load_balancer(config.get('region'),config.get('elb_name')) elb.deregister_instances([id]) #stop it instance = aws_ec2.get_instance(config,id) instance.stop() #Update config instances = config.get_list('instances') instances.remove(id) config.set_list('instances',instances) logger.info('Unhealthy instance is successfully stopped. ') logger.info('A new instance instead of stopped one will be allocated automatically.') else: unhealthy_instances[id] = num_checks + 1 else: unhealthy_instances[id] = 1 logger.warning('Instance %s is unhealthy with state %s and code %s' %(id, state.state, state.reason_code)) logger.warning('After %s checks it will be stopped.' % static.HEALTH_CHECKS)
def scale_up_with_new_instance(config): """Scale up with a new instance that first gets configured""" logger = logging.getLogger(__name__) logger.info("Upscaling with the new instance started.") # Launch instance logger.info("Launching an instance.") instance, _ = aws_ec2.launch_instance() # Start app logger.info("Performing deployment operations.") aws_ec2.app_predeployment(config) aws_ec2.deploy_app_at_instance(config, instance) aws_ec2.app_postdeployment(config) # Register instance at load balancer logger.info("Registering an instance at the load balancer") elb = aws_ec2_elb.get_load_balancer(config.get("region"), config.get("elb_name")) elb.register_instances([instance.id]) # Update config instances = config.get_list("instances") instances.append(instance.id) config.set_list("instances", instances)
def delete_environment(config): """Delete an environment given its ID""" logger = logging.getLogger(__name__) logger.info('Deleting environment %s' % config.env_id) instances = config.get('instances').split(',') try: stopped_instances = config.get('stopped_instances').split(',') except ConfigParser.NoOptionError: stopped_instances = [] logger.info('Deleting load balancer') elb_name = config.get('elb_name') region = config.get('region') lb = aws_ec2_elb.get_load_balancer(region, elb_name) lb.delete() logger.info('Terminating instances') aws_ec2.terminate_instances(instances) if len(stopped_instances) > 0: aws_ec2.terminate_instances(stopped_instances) config.remove_section(env_id) output = 'Environment %s deleted, %d instance(s) terminated' %(env_id, len(instances) + len(stopped_instances)) logger.info(output) print output return config
def scale_up_from_pool(config): """Scale up an instance from an available pool of previously stopped instances""" logger = logging.getLogger(__name__) # Launch instance logger.info("Upscaling from the pool started.") stopped_instance_id = config.get_list("stopped_instances")[-1] instance = aws_ec2.get_instance(config, stopped_instance_id) instance.start() time.sleep(static.AUTOSCALE_DELAY_AFTER_START) # Start httpd service # Do not send instance directly as it does not have public DNS attached now, need to get it again from EC2 aws_ec2.start_httpd(config, instance.id) # Register instance at load balancer logger.info("Registering an instance at the load balancer") elb = aws_ec2_elb.get_load_balancer(config.get("region"), config.get("elb_name")) elb.register_instances([instance.id]) # Update config instances = config.get_list("stopped_instances") instances.remove(instance.id) config.set_list("stopped_instances", instances) instances = config.get_list("instances") instances.append(instance.id) config.set_list("instances", instances)
threading.Thread(target=scale.scale_up, args=(config,)).start() set_next_possible_scaling_time() else: logger.info('Next upscaling is deferred until getting statistics from a newly added instance.') else: logger.warning('The environment has exceeded scaling capacity but further scaling needed.') else: logger.warning('The environment is highly loaded, scaling needed.') elif percentage < static.AUTOSCALE_CPU_PERCENTAGE_DOWN: if not args.noscale: if scale.scaling_down_possible(config): if is_downscaling_allowed(): # least_loaded_instance = avg_cpu.get_least_loaded_instance() # logger.info('Least loaded instance is %s' %least_loaded_instance) threading.Thread(target=scale.scale_down, args=(config,config.get_list('instances')[-1])).start() set_next_possible_scaling_time() else: logger.info('Next downscaling is deferred until getting statistics from an updated environment.') else: if scale.scaling_down_possible(config): logger.info('The load on the environment is low, downscaling is possible.') #Health check elb = aws_ec2_elb.get_load_balancer(config.get('region'),config.get('elb_name')) instance_states = elb.get_instance_health() for instance_state in instance_states: process_instance_state(config, instance_state) logger.info('Start waiting') time.sleep(static.MONITOR_SLEEP_TIME) logger.info('Stop waiting')