def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_table_option(key_name, 'enable_reads_autoscaling'): logger.info( '{0} - Autoscaling of reads has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_read_units(table_name), 0 update_needed = False try: lookback_window_start = get_table_option(key_name, 'lookback_window_start') lookback_period = get_table_option(key_name, 'lookback_period') current_read_units = dynamodb.get_provisioned_table_read_units( table_name) consumed_read_units_percent = \ table_stats.get_consumed_read_units_percent( table_name, lookback_window_start, lookback_period) throttled_read_count = \ table_stats.get_throttled_read_event_count( table_name, lookback_window_start, lookback_period) throttled_by_provisioned_read_percent = \ table_stats.get_throttled_by_provisioned_read_event_percent( table_name, lookback_window_start, lookback_period) throttled_by_consumed_read_percent = \ table_stats.get_throttled_by_consumed_read_percent( table_name, lookback_window_start, lookback_period) reads_upper_threshold = \ get_table_option(key_name, 'reads_upper_threshold') reads_lower_threshold = \ get_table_option(key_name, 'reads_lower_threshold') throttled_reads_upper_threshold = \ get_table_option(key_name, 'throttled_reads_upper_threshold') increase_reads_with = \ get_table_option(key_name, 'increase_reads_with') increase_reads_unit = \ get_table_option(key_name, 'increase_reads_unit') decrease_reads_with = \ get_table_option(key_name, 'decrease_reads_with') decrease_reads_unit = \ get_table_option(key_name, 'decrease_reads_unit') min_provisioned_reads = \ get_table_option(key_name, 'min_provisioned_reads') max_provisioned_reads = \ get_table_option(key_name, 'max_provisioned_reads') num_read_checks_before_scale_down = \ get_table_option(key_name, 'num_read_checks_before_scale_down') num_read_checks_reset_percent = \ get_table_option(key_name, 'num_read_checks_reset_percent') increase_throttled_by_provisioned_reads_unit = \ get_table_option( key_name, 'increase_throttled_by_provisioned_reads_unit') increase_throttled_by_provisioned_reads_scale = \ get_table_option( key_name, 'increase_throttled_by_provisioned_reads_scale') increase_throttled_by_consumed_reads_unit = \ get_table_option( key_name, 'increase_throttled_by_consumed_reads_unit') increase_throttled_by_consumed_reads_scale = \ get_table_option( key_name, 'increase_throttled_by_consumed_reads_scale') increase_consumed_reads_unit = \ get_table_option(key_name, 'increase_consumed_reads_unit') increase_consumed_reads_with = \ get_table_option(key_name, 'increase_consumed_reads_with') increase_consumed_reads_scale = \ get_table_option(key_name, 'increase_consumed_reads_scale') decrease_consumed_reads_unit = \ get_table_option(key_name, 'decrease_consumed_reads_unit') decrease_consumed_reads_with = \ get_table_option(key_name, 'decrease_consumed_reads_with') decrease_consumed_reads_scale = \ get_table_option(key_name, 'decrease_consumed_reads_scale') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units # Reset consecutive reads if num_read_checks_reset_percent is reached if num_read_checks_reset_percent: if consumed_read_units_percent >= num_read_checks_reset_percent: logger.info('{0} - Resetting the number of consecutive ' 'read checks. Reason: Consumed percent {1} is ' 'greater than reset percent: {2}'.format( table_name, consumed_read_units_percent, num_read_checks_reset_percent)) num_consec_read_checks = 0 # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_reads_up_scaling'): logger.debug( '{0} - Up scaling event detected. No action taken as scaling ' 'up reads has been disabled in the configuration'.format( table_name)) else: # If local/granular values not specified use global values increase_consumed_reads_unit = \ increase_consumed_reads_unit or increase_reads_unit increase_throttled_by_provisioned_reads_unit = \ increase_throttled_by_provisioned_reads_unit or increase_reads_unit increase_throttled_by_consumed_reads_unit = \ increase_throttled_by_consumed_reads_unit or increase_reads_unit increase_consumed_reads_with = \ increase_consumed_reads_with or increase_reads_with # Initialise variables to store calculated provisioning throttled_by_provisioned_calculated_provisioning = scale_reader( increase_throttled_by_provisioned_reads_scale, throttled_by_provisioned_read_percent) throttled_by_consumed_calculated_provisioning = scale_reader( increase_throttled_by_consumed_reads_scale, throttled_by_consumed_read_percent) consumed_calculated_provisioning = scale_reader( increase_consumed_reads_scale, consumed_read_units_percent) throttled_count_calculated_provisioning = 0 calculated_provisioning = 0 # Increase needed due to high throttled to provisioned ratio if throttled_by_provisioned_calculated_provisioning: if increase_throttled_by_provisioned_reads_unit == 'percent': throttled_by_provisioned_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: throttled_by_provisioned_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Increase needed due to high throttled to consumed ratio if throttled_by_consumed_calculated_provisioning: if increase_throttled_by_consumed_reads_unit == 'percent': throttled_by_consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: throttled_by_consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Increase needed due to high CU consumption if consumed_calculated_provisioning: if increase_consumed_reads_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) elif (reads_upper_threshold and consumed_read_units_percent > reads_upper_threshold and not increase_consumed_reads_scale): if increase_consumed_reads_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, increase_consumed_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, increase_consumed_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Increase needed due to high throttling if (throttled_reads_upper_threshold and throttled_read_count > throttled_reads_upper_threshold): if increase_reads_unit == 'percent': throttled_count_calculated_provisioning = \ calculators.increase_reads_in_percent( updated_read_units, increase_consumed_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: throttled_count_calculated_provisioning = \ calculators.increase_reads_in_units( updated_read_units, increase_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Determine which metric requires the most scaling if (throttled_by_provisioned_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_provisioned_calculated_provisioning scale_reason = ("due to throttled events by provisioned " "units threshold being exceeded") if (throttled_by_consumed_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_consumed_calculated_provisioning scale_reason = ("due to throttled events by consumed " "units threshold being exceeded") if consumed_calculated_provisioning > calculated_provisioning: calculated_provisioning = consumed_calculated_provisioning scale_reason = "due to consumed threshold being exceeded" if throttled_count_calculated_provisioning > calculated_provisioning: calculated_provisioning = throttled_count_calculated_provisioning scale_reason = "due to throttled events threshold being exceeded" if calculated_provisioning > current_read_units: logger.info('{0} - Resetting the number of consecutive ' 'read checks. Reason: scale up {1}'.format( table_name, scale_reason)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Decrease needed due to low CU consumption if not update_needed: # If local/granular values not specified use global values decrease_consumed_reads_unit = \ decrease_consumed_reads_unit or decrease_reads_unit decrease_consumed_reads_with = \ decrease_consumed_reads_with or decrease_reads_with # Initialise variables to store calculated provisioning consumed_calculated_provisioning = scale_reader_decrease( decrease_consumed_reads_scale, consumed_read_units_percent) calculated_provisioning = None # Exit if down scaling has been disabled if not get_table_option(key_name, 'enable_reads_down_scaling'): logger.debug( '{0} - Down scaling event detected. No action taken as scaling' ' down reads has been disabled in the configuration'.format( table_name)) # Exit if reads == 0% and downscaling has been disabled at 0% elif (consumed_read_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - Down scaling event detected. No action taken as scaling' ' down reads is not done when usage is at 0%'.format( table_name)) else: if consumed_calculated_provisioning: if decrease_consumed_reads_unit == 'percent': calculated_provisioning = \ calculators.decrease_reads_in_percent( updated_read_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_reads'), table_name) else: calculated_provisioning = \ calculators.decrease_reads_in_units( updated_read_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_reads'), table_name) elif (reads_lower_threshold and consumed_read_units_percent < reads_lower_threshold and not decrease_consumed_reads_scale): if decrease_consumed_reads_unit == 'percent': calculated_provisioning = \ calculators.decrease_reads_in_percent( updated_read_units, decrease_consumed_reads_with, get_table_option( key_name, 'min_provisioned_reads'), table_name) else: calculated_provisioning = \ calculators.decrease_reads_in_units( updated_read_units, decrease_consumed_reads_with, get_table_option( key_name, 'min_provisioned_reads'), table_name) if (calculated_provisioning and current_read_units != calculated_provisioning): num_consec_read_checks += 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info('Will not increase writes over max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) # Ensure that we have met the min-provisioning if min_provisioned_reads: if int(min_provisioned_reads) > int(updated_read_units): update_needed = True updated_read_units = int(min_provisioned_reads) logger.info('{0} - Increasing reads to meet min-provisioned-reads ' 'limit ({1} reads)'.format(table_name, updated_read_units)) if calculators.is_consumed_over_proposed(current_read_units, updated_read_units, consumed_read_units_percent): update_needed = False updated_read_units = current_read_units logger.info( '{0} - Consumed is over proposed read units. Will leave table at ' 'current setting.'.format(table_name)) logger.info('{0} - Consecutive read checks {1}/{2}'.format( table_name, num_consec_read_checks, num_read_checks_before_scale_down)) return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_table_option(key_name, 'enable_reads_autoscaling'): logger.info( '{0} - Autoscaling of reads has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_read_units(table_name), 0 update_needed = False try: lookback_window_start = get_table_option( key_name, 'lookback_window_start') lookback_period = get_table_option(key_name, 'lookback_period') current_read_units = dynamodb.get_provisioned_table_read_units( table_name) consumed_read_units_percent = \ table_stats.get_consumed_read_units_percent( table_name, lookback_window_start, lookback_period) throttled_read_count = \ table_stats.get_throttled_read_event_count( table_name, lookback_window_start, lookback_period) throttled_by_provisioned_read_percent = \ table_stats.get_throttled_by_provisioned_read_event_percent( table_name, lookback_window_start, lookback_period) throttled_by_consumed_read_percent = \ table_stats.get_throttled_by_consumed_read_percent( table_name, lookback_window_start, lookback_period) reads_upper_threshold = \ get_table_option(key_name, 'reads_upper_threshold') reads_lower_threshold = \ get_table_option(key_name, 'reads_lower_threshold') throttled_reads_upper_threshold = \ get_table_option(key_name, 'throttled_reads_upper_threshold') increase_reads_with = \ get_table_option(key_name, 'increase_reads_with') increase_reads_unit = \ get_table_option(key_name, 'increase_reads_unit') decrease_reads_with = \ get_table_option(key_name, 'decrease_reads_with') decrease_reads_unit = \ get_table_option(key_name, 'decrease_reads_unit') min_provisioned_reads = \ get_table_option(key_name, 'min_provisioned_reads') max_provisioned_reads = \ get_table_option(key_name, 'max_provisioned_reads') num_read_checks_before_scale_down = \ get_table_option(key_name, 'num_read_checks_before_scale_down') num_read_checks_reset_percent = \ get_table_option(key_name, 'num_read_checks_reset_percent') increase_throttled_by_provisioned_reads_unit = \ get_table_option( key_name, 'increase_throttled_by_provisioned_reads_unit') increase_throttled_by_provisioned_reads_scale = \ get_table_option( key_name, 'increase_throttled_by_provisioned_reads_scale') increase_throttled_by_consumed_reads_unit = \ get_table_option( key_name, 'increase_throttled_by_consumed_reads_unit') increase_throttled_by_consumed_reads_scale = \ get_table_option( key_name, 'increase_throttled_by_consumed_reads_scale') increase_consumed_reads_unit = \ get_table_option(key_name, 'increase_consumed_reads_unit') increase_consumed_reads_with = \ get_table_option(key_name, 'increase_consumed_reads_with') increase_consumed_reads_scale = \ get_table_option(key_name, 'increase_consumed_reads_scale') decrease_consumed_reads_unit = \ get_table_option(key_name, 'decrease_consumed_reads_unit') decrease_consumed_reads_with = \ get_table_option(key_name, 'decrease_consumed_reads_with') decrease_consumed_reads_scale = \ get_table_option(key_name, 'decrease_consumed_reads_scale') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units # Reset consecutive reads if num_read_checks_reset_percent is reached if num_read_checks_reset_percent: if consumed_read_units_percent >= num_read_checks_reset_percent: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: Consumed percent {1} is ' 'greater than reset percent: {2}'.format( table_name, consumed_read_units_percent, num_read_checks_reset_percent)) num_consec_read_checks = 0 # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_reads_up_scaling'): logger.debug( '{0} - Up scaling event detected. No action taken as scaling ' 'up reads has been disabled in the configuration'.format( table_name)) else: # If local/granular values not specified use global values increase_consumed_reads_unit = \ increase_consumed_reads_unit or increase_reads_unit increase_throttled_by_provisioned_reads_unit = \ increase_throttled_by_provisioned_reads_unit or increase_reads_unit increase_throttled_by_consumed_reads_unit = \ increase_throttled_by_consumed_reads_unit or increase_reads_unit increase_consumed_reads_with = \ increase_consumed_reads_with or increase_reads_with # Initialise variables to store calculated provisioning throttled_by_provisioned_calculated_provisioning = scale_reader( increase_throttled_by_provisioned_reads_scale, throttled_by_provisioned_read_percent) throttled_by_consumed_calculated_provisioning = scale_reader( increase_throttled_by_consumed_reads_scale, throttled_by_consumed_read_percent) consumed_calculated_provisioning = scale_reader( increase_consumed_reads_scale, consumed_read_units_percent) throttled_count_calculated_provisioning = 0 calculated_provisioning = 0 # Increase needed due to high throttled to provisioned ratio if throttled_by_provisioned_calculated_provisioning: if increase_throttled_by_provisioned_reads_unit == 'percent': throttled_by_provisioned_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: throttled_by_provisioned_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Increase needed due to high throttled to consumed ratio if throttled_by_consumed_calculated_provisioning: if increase_throttled_by_consumed_reads_unit == 'percent': throttled_by_consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: throttled_by_consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Increase needed due to high CU consumption if consumed_calculated_provisioning: if increase_consumed_reads_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) elif (reads_upper_threshold and consumed_read_units_percent > reads_upper_threshold and not increase_consumed_reads_scale): if increase_consumed_reads_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, increase_consumed_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, increase_consumed_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Increase needed due to high throttling if (throttled_reads_upper_threshold and throttled_read_count > throttled_reads_upper_threshold): if increase_reads_unit == 'percent': throttled_count_calculated_provisioning = \ calculators.increase_reads_in_percent( updated_read_units, increase_consumed_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) else: throttled_count_calculated_provisioning = \ calculators.increase_reads_in_units( updated_read_units, increase_reads_with, get_table_option(key_name, 'max_provisioned_reads'), consumed_read_units_percent, table_name) # Determine which metric requires the most scaling if (throttled_by_provisioned_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_provisioned_calculated_provisioning scale_reason = ( "due to throttled events by provisioned " "units threshold being exceeded") if (throttled_by_consumed_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_consumed_calculated_provisioning scale_reason = ( "due to throttled events by consumed " "units threshold being exceeded") if consumed_calculated_provisioning > calculated_provisioning: calculated_provisioning = consumed_calculated_provisioning scale_reason = "due to consumed threshold being exceeded" if throttled_count_calculated_provisioning > calculated_provisioning: calculated_provisioning = throttled_count_calculated_provisioning scale_reason = "due to throttled events threshold being exceeded" if calculated_provisioning > current_read_units: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: scale up {1}'.format( table_name, scale_reason)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Decrease needed due to low CU consumption if not update_needed: # If local/granular values not specified use global values decrease_consumed_reads_unit = \ decrease_consumed_reads_unit or decrease_reads_unit decrease_consumed_reads_with = \ decrease_consumed_reads_with or decrease_reads_with # Initialise variables to store calculated provisioning consumed_calculated_provisioning = scale_reader_decrease( decrease_consumed_reads_scale, consumed_read_units_percent) calculated_provisioning = None # Exit if down scaling has been disabled if not get_table_option(key_name, 'enable_reads_down_scaling'): logger.debug( '{0} - Down scaling event detected. No action taken as scaling' ' down reads has been disabled in the configuration'.format( table_name)) # Exit if reads == 0% and downscaling has been disabled at 0% elif (consumed_read_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - Down scaling event detected. No action taken as scaling' ' down reads is not done when usage is at 0%'.format( table_name)) else: if consumed_calculated_provisioning: if decrease_consumed_reads_unit == 'percent': calculated_provisioning = \ calculators.decrease_reads_in_percent( updated_read_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_reads'), table_name) else: calculated_provisioning = \ calculators.decrease_reads_in_units( updated_read_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_reads'), table_name) elif (reads_lower_threshold and consumed_read_units_percent < reads_lower_threshold and not decrease_consumed_reads_scale): if decrease_consumed_reads_unit == 'percent': calculated_provisioning = \ calculators.decrease_reads_in_percent( updated_read_units, decrease_consumed_reads_with, get_table_option( key_name, 'min_provisioned_reads'), table_name) else: calculated_provisioning = \ calculators.decrease_reads_in_units( updated_read_units, decrease_consumed_reads_with, get_table_option( key_name, 'min_provisioned_reads'), table_name) if (calculated_provisioning and current_read_units != calculated_provisioning): num_consec_read_checks += 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( 'Will not increase writes over max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) # Ensure that we have met the min-provisioning if min_provisioned_reads: if int(min_provisioned_reads) > int(updated_read_units): update_needed = True updated_read_units = int(min_provisioned_reads) logger.info( '{0} - Increasing reads to meet min-provisioned-reads ' 'limit ({1} reads)'.format(table_name, updated_read_units)) if calculators.is_consumed_over_proposed( current_read_units, updated_read_units, consumed_read_units_percent): update_needed = False updated_read_units = current_read_units logger.info( '{0} - Consumed is over proposed read units. Will leave table at ' 'current setting.'.format(table_name)) logger.info('{0} - Consecutive read checks {1}/{2}'.format( table_name, num_consec_read_checks, num_read_checks_before_scale_down)) return update_needed, updated_read_units, num_consec_read_checks