def scale_up_asg(cluster_name, asg, count): asg_old_max_size = asg['MaxSize'] asg_old_desired_capacity = asg['DesiredCapacity'] desired_capacity = asg_old_desired_capacity + count asg_tags = asg['Tags'] asg_name = asg['AutoScalingGroupName'] # remove any stale suspensions from asg that may be present modify_aws_autoscaling(asg_name, "resume") asg_tag_desired_capacity = get_asg_tag(asg_tags, app_config["ASG_DESIRED_STATE_TAG"]) asg_tag_orig_capacity = get_asg_tag(asg_tags, app_config["ASG_ORIG_CAPACITY_TAG"]) asg_tag_orig_max_capacity = get_asg_tag( asg_tags, app_config["ASG_ORIG_MAX_CAPACITY_TAG"]) if desired_capacity == asg_old_desired_capacity: logger.info( f'Desired and current capacity for {asg_name} are equal. Skipping ASG.' ) if asg_tag_desired_capacity.get('Value'): logger.info( 'Found capacity tags on ASG from previous run. Leaving alone.') return int(asg_tag_desired_capacity.get('Value')), int( asg_tag_orig_capacity.get('Value')), int( asg_tag_orig_max_capacity.get('Value')) else: save_asg_tags(asg_name, app_config["ASG_ORIG_CAPACITY_TAG"], asg_old_desired_capacity) save_asg_tags(asg_name, app_config["ASG_DESIRED_STATE_TAG"], asg_old_desired_capacity) save_asg_tags(asg_name, app_config["ASG_ORIG_MAX_CAPACITY_TAG"], asg_old_max_size) return asg_old_desired_capacity, asg_old_desired_capacity, asg_old_max_size # only scale up if no previous desired capacity tag set if asg_tag_desired_capacity.get('Value'): logger.info( 'Found previous desired capacity value tag set on asg from a previous run.' ) logger.info( f'Maintaining previous capacity of {asg_old_desired_capacity} to not overscale.' ) asg_instance_count = count_all_cluster_instances(cluster_name) # check cluster health before doing anything if not validate_cluster_health( asg_name, int(asg_tag_desired_capacity.get('Value')), asg_instance_count): logger.info('Exiting since ASG healthcheck failed') raise Exception('ASG healthcheck failed') return int(asg_tag_desired_capacity.get('Value')), int( asg_tag_orig_capacity.get('Value')), int( asg_tag_orig_max_capacity.get('Value')) else: logger.info( 'No previous capacity value tags set on ASG; setting tags.') save_asg_tags(asg_name, app_config["ASG_ORIG_CAPACITY_TAG"], asg_old_desired_capacity) save_asg_tags(asg_name, app_config["ASG_DESIRED_STATE_TAG"], desired_capacity) save_asg_tags(asg_name, app_config["ASG_ORIG_MAX_CAPACITY_TAG"], asg_old_max_size) # only change the max size if the new capacity is bigger than current max if desired_capacity > asg_old_max_size: scale_asg(asg_name, asg_old_desired_capacity, desired_capacity, desired_capacity) else: scale_asg(asg_name, asg_old_desired_capacity, desired_capacity, asg_old_max_size) cluster_health_wait = app_config['CLUSTER_HEALTH_WAIT'] logger.info( f'Waiting for {cluster_health_wait} seconds for ASG to scale before validating cluster health...' ) time.sleep(cluster_health_wait) asg_instance_count = count_all_cluster_instances(cluster_name) # check cluster health before doing anything if not validate_cluster_health(asg_name, desired_capacity, asg_instance_count): logger.info('Exiting since ASG healthcheck failed') raise Exception('ASG healthcheck failed') return desired_capacity, asg_old_desired_capacity, asg_old_max_size
def test_count_all_cluster_instances(self): count = count_all_cluster_instances('mock-cluster') self.assertEqual(count, 3)
def update_asgs(asgs, cluster_name): for asg in asgs: logger.info('\n') logger.info( '**** Starting rolling update for autoscaling group {} ****'. format(asg['AutoScalingGroupName'])) asg_name = asg['AutoScalingGroupName'] asg_lc_name = asg['LaunchConfigurationName'] asg_old_max_size = asg['MaxSize'] instances = asg['Instances'] asg_old_desired_capacity = asg['DesiredCapacity'] asg_tags = asg['Tags'] # return a list of outdated instances outdated_instances = [] for instance in instances: if instance_outdated(instance, asg_lc_name): outdated_instances.append(instance) logger.info('Found {} outdated instances'.format( len(outdated_instances))) # skip to next asg if there are no outdated instances if len(outdated_instances) == 0: continue # remove any stale suspentions from asg that may be present modify_aws_autoscaling(asg_name, "resume") # check for previous run tag on asg asg_tag_desired_capacity = get_asg_tag( asg_tags, app_config["ASG_DESIRED_STATE_TAG"]) if asg_tag_desired_capacity.get('Value'): logger.info( 'Found previous desired capacity value tag set on asg from a previous run. Value: {}' .format(asg_tag_desired_capacity.get('Value'))) logger.info('Maintaining previous capacity to not overscale') asg_new_desired_capacity = int( asg_tag_desired_capacity.get('Value')) asg_tag_original_capacity = get_asg_tag( asg_tags, app_config["ASG_ORIG_CAPACITY_TAG"]) logger.info( 'Maintaining original old capacity from a previous run so we can scale back down to original size of: {}' .format(asg_tag_original_capacity.get('Value'))) asg_old_desired_capacity = int( asg_tag_original_capacity.get('Value')) else: logger.info('No previous capacity value tag set on asg') # save original capacity to asg tags logger.info('Setting original capacity on asg') save_asg_tags(asg_name, app_config["ASG_ORIG_CAPACITY_TAG"], asg_old_desired_capacity) asg_new_desired_capacity = asg_old_desired_capacity + len( outdated_instances) # save new capacity to asg tags save_asg_tags(asg_name, app_config["ASG_DESIRED_STATE_TAG"], asg_new_desired_capacity) # only change the max size if the new capacity is bigger than current max if asg_new_desired_capacity > asg_old_max_size: asg_new_max_size = asg_new_desired_capacity else: # dont change the size asg_new_max_size = asg_old_max_size # get number of k8s nodes before we scale used later # to determine how many new nodes have been created k8s_nodes = get_k8s_nodes() # now scale up scale_asg(asg_name, asg_old_desired_capacity, asg_new_desired_capacity, asg_new_max_size) logger.info( 'Waiting for {} seconds for asg {} to scale before validating cluster health...' .format(app_config['CLUSTER_HEALTH_WAIT'], asg_name)) time.sleep(app_config['CLUSTER_HEALTH_WAIT']) # check how many instances are running asg_instance_count = count_all_cluster_instances(cluster_name) # check cluster health before doing anything if validate_cluster_health(asg_name, asg_new_desired_capacity, asg_instance_count): # pause aws autoscaling so new instances dont try # to spawn while instances are being terminated modify_aws_autoscaling(asg_name, "suspend") # start draining and terminating for outdated in outdated_instances: # catch any failures so we can resume aws autoscaling try: # get the k8s node name instead of instance id node_name = get_node_by_instance_id( k8s_nodes, outdated['InstanceId']) drain_node(node_name) delete_node(node_name) terminate_instance(outdated['InstanceId']) if not instance_terminated(outdated['InstanceId']): raise Exception( 'Instance is failing to terminate. Cancelling out.' ) detach_instance(outdated['InstanceId'], asg_name) if not instance_detached(outdated['InstanceId']): raise Exception( 'Instance is failing to detach from ASG. Cancelling out.' ) except Exception as e: logger.info(e) raise RollingUpdateException( "Rolling update on asg failed", asg_name) # resume aws autoscaling modify_aws_autoscaling(asg_name, "resume") # remove aws tag delete_asg_tags(asg_name, app_config["ASG_DESIRED_STATE_TAG"]) delete_asg_tags(asg_name, app_config["ASG_ORIG_CAPACITY_TAG"]) logger.info('*** Rolling update of asg {} is complete! ***'.format( asg_name)) else: logger.info('Exiting since asg healthcheck failed') raise Exception('Asg healthcheck failed') logger.info('All asgs processed')
def test_count_all_cluster_instances_fail(self): count = count_all_cluster_instances('foo-bar') self.assertEqual(count, 0)