def scale_down(config, instance_id):
"""Remove one instance from the environment and put it to the pool of stopped machines"""
logger = logging.getLogger(__name__)
if not scaling_down_possible(config):
logger.warning("Scaling down not possible, minimum number of instances is running")
else:
logger.info("Downscaling started.")
# Deregister it from load balancer
elb = aws_ec2_elb.get_load_balancer(config.get("region"), config.get("elb_name"))
elb.deregister_instances([instance_id])
# stop it
time.sleep(static.AUTOSCALE_DELAY_BEFORE_REMOVING_INSTANCE) # let instance respond to pending requests
instance = aws_ec2.get_instance(config, instance_id)
instance.stop()
# Update config
try:
instances = config.get_list("stopped_instances")
except ConfigParser.NoOptionError:
instances = []
instances.append(instance_id)
config.set_list("stopped_instances", instances)
instances = config.get_list("instances")
instances.remove(instance_id)
config.set_list("instances", instances)
logger.info("Downscaling completed. 1 instance stopped and is available in the pool.")
if len(instances) > 1:
logger.info("%d instances are currently on duty." % len(instances))
else:
logger.info("1 instance is currently on duty.")
def process_instance_state(config, state):
if state.instance_id in config.get_list('instances') and \
not state.state == 'InService' and \
not state.reason_code == 'Instance registration is still in progress.' and \
not state.reason_code == 'ELB':
id = state.instance_id
if unhealthy_instances.has_key(id):
num_checks = unhealthy_instances.get(id)
if num_checks + 1 == static.HEALTH_CHECKS:
logger.info('Stopping unhealty instance %s for investigation' %id)
unhealthy_instances.pop(id)
#Instance is considered failed and should be stopped for investigation
#Deregister it from load balancer
elb = aws_ec2_elb.get_load_balancer(config.get('region'),config.get('elb_name'))
elb.deregister_instances([id])
#stop it
instance = aws_ec2.get_instance(config,id)
instance.stop()
#Update config
instances = config.get_list('instances')
instances.remove(id)
config.set_list('instances',instances)
logger.info('Unhealthy instance is successfully stopped. ')
logger.info('A new instance instead of stopped one will be allocated automatically.')
else:
unhealthy_instances[id] = num_checks + 1
else:
unhealthy_instances[id] = 1
logger.warning('Instance %s is unhealthy with state %s and code %s' %(id, state.state, state.reason_code))
logger.warning('After %s checks it will be stopped.' % static.HEALTH_CHECKS)
def scale_up_with_new_instance(config):
"""Scale up with a new instance that first gets configured"""
logger = logging.getLogger(__name__)
logger.info("Upscaling with the new instance started.")
# Launch instance
logger.info("Launching an instance.")
instance, _ = aws_ec2.launch_instance()
# Start app
logger.info("Performing deployment operations.")
aws_ec2.app_predeployment(config)
aws_ec2.deploy_app_at_instance(config, instance)
aws_ec2.app_postdeployment(config)
# Register instance at load balancer
logger.info("Registering an instance at the load balancer")
elb = aws_ec2_elb.get_load_balancer(config.get("region"), config.get("elb_name"))
elb.register_instances([instance.id])
# Update config
instances = config.get_list("instances")
instances.append(instance.id)
config.set_list("instances", instances)
def delete_environment(config):
"""Delete an environment given its ID"""
logger = logging.getLogger(__name__)
logger.info('Deleting environment %s' % config.env_id)
instances = config.get('instances').split(',')
try:
stopped_instances = config.get('stopped_instances').split(',')
except ConfigParser.NoOptionError:
stopped_instances = []
logger.info('Deleting load balancer')
elb_name = config.get('elb_name')
region = config.get('region')
lb = aws_ec2_elb.get_load_balancer(region, elb_name)
lb.delete()
logger.info('Terminating instances')
aws_ec2.terminate_instances(instances)
if len(stopped_instances) > 0:
aws_ec2.terminate_instances(stopped_instances)
config.remove_section(env_id)
output = 'Environment %s deleted, %d instance(s) terminated' %(env_id, len(instances) + len(stopped_instances))
logger.info(output)
print output
return config
def scale_up_from_pool(config):
"""Scale up an instance from an available pool of previously stopped instances"""
logger = logging.getLogger(__name__)
# Launch instance
logger.info("Upscaling from the pool started.")
stopped_instance_id = config.get_list("stopped_instances")[-1]
instance = aws_ec2.get_instance(config, stopped_instance_id)
instance.start()
time.sleep(static.AUTOSCALE_DELAY_AFTER_START)
# Start httpd service
# Do not send instance directly as it does not have public DNS attached now, need to get it again from EC2
aws_ec2.start_httpd(config, instance.id)
# Register instance at load balancer
logger.info("Registering an instance at the load balancer")
elb = aws_ec2_elb.get_load_balancer(config.get("region"), config.get("elb_name"))
elb.register_instances([instance.id])
# Update config
instances = config.get_list("stopped_instances")
instances.remove(instance.id)
config.set_list("stopped_instances", instances)
instances = config.get_list("instances")
instances.append(instance.id)
config.set_list("instances", instances)
threading.Thread(target=scale.scale_up, args=(config,)).start()
set_next_possible_scaling_time()
else:
logger.info('Next upscaling is deferred until getting statistics from a newly added instance.')
else:
logger.warning('The environment has exceeded scaling capacity but further scaling needed.')
else:
logger.warning('The environment is highly loaded, scaling needed.')
elif percentage < static.AUTOSCALE_CPU_PERCENTAGE_DOWN:
if not args.noscale:
if scale.scaling_down_possible(config):
if is_downscaling_allowed():
# least_loaded_instance = avg_cpu.get_least_loaded_instance()
# logger.info('Least loaded instance is %s' %least_loaded_instance)
threading.Thread(target=scale.scale_down, args=(config,config.get_list('instances')[-1])).start()
set_next_possible_scaling_time()
else:
logger.info('Next downscaling is deferred until getting statistics from an updated environment.')
else:
if scale.scaling_down_possible(config):
logger.info('The load on the environment is low, downscaling is possible.')
#Health check
elb = aws_ec2_elb.get_load_balancer(config.get('region'),config.get('elb_name'))
instance_states = elb.get_instance_health()
for instance_state in instance_states:
process_instance_state(config, instance_state)
logger.info('Start waiting')
time.sleep(static.MONITOR_SLEEP_TIME)
logger.info('Stop waiting')
