def __ensure_provisioning_writes( table_name, table_key, gsi_name, gsi_key, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'): logger.info( '{0} - GSI: {1} - ' 'Autoscaling of writes has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name), 0 update_needed = False try: current_write_units = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) consumed_write_units_percent = \ gsi_stats.get_consumed_write_units_percent(table_name, gsi_name) throttled_write_count = \ gsi_stats.get_throttled_write_event_count(table_name, gsi_name) writes_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'writes_upper_threshold') writes_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_gsi_option(table_key, gsi_key, 'increase_writes_unit') increase_writes_with = \ get_gsi_option(table_key, gsi_key, 'increase_writes_with') decrease_writes_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_writes_unit') decrease_writes_with = \ get_gsi_option(table_key, gsi_key, 'decrease_writes_with') max_provisioned_writes = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_writes') num_write_checks_before_scale_down = \ get_gsi_option( table_key, gsi_key, 'num_write_checks_before_scale_down') num_write_checks_reset_percent = \ get_gsi_option(key_name, 'num_write_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current write unit value updated_write_units = current_write_units # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down writes is not done when usage is at 0%'.format( table_name, gsi_name)) elif consumed_write_units_percent >= writes_upper_threshold: if increase_writes_unit == 'percent': calulated_provisioning = calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calulated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_write_units != calulated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calulated_provisioning elif throttled_write_count > throttled_writes_upper_threshold: if throttled_writes_upper_threshold > 0: if increase_writes_unit == 'percent': calulated_provisioning = calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calulated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_write_units != calulated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calulated_provisioning elif consumed_write_units_percent >= num_write_checks_reset_percent: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format( table_name, consumed_write_units_percent, num_write_checks_reset_percent)) num_consec_write_checks = 0 elif consumed_write_units_percent <= writes_lower_threshold: if decrease_writes_unit == 'percent': calulated_provisioning = calculators.decrease_writes_in_percent( current_write_units, decrease_writes_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calulated_provisioning = calculators.decrease_writes_in_units( current_write_units, decrease_writes_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_write_units != calulated_provisioning: num_consec_write_checks = num_consec_write_checks + 1 if num_consec_write_checks >= num_write_checks_before_scale_down: update_needed = True updated_write_units = calulated_provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info( '{0} - GSI: {1} - ' 'Will not increase writes over gsi-max-provisioned-writes ' 'limit ({2} writes)'.format( table_name, gsi_name, updated_write_units)) logger.info('{0} - Consecutive write checks {1}/{2}'.format( table_name, num_consec_write_checks, num_write_checks_before_scale_down)) return update_needed, updated_write_units, num_consec_write_checks
def test_decrease_writes_in_units(self): """ Ensure that a regular decrease works """ result = calculators.decrease_writes_in_units(200, 90, 1, 'test') self.assertEqual(result, 110)
def test_decrease_writes_in_units_hit_min_value(self): """ Check that min values are honoured """ result = calculators.decrease_writes_in_units(20, 50, 15, 'test') self.assertEqual(result, 15)
def __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_table_option(key_name, 'enable_writes_autoscaling'): logger.info( '{0} - Autoscaling of writes has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_write_units(table_name), 0 update_needed = False try: lookback_window_start = get_table_option( key_name, 'lookback_window_start') lookback_period = get_table_option(key_name, 'lookback_period') current_write_units = dynamodb.get_provisioned_table_write_units( table_name) consumed_write_units_percent = \ table_stats.get_consumed_write_units_percent( table_name, lookback_window_start, lookback_period) throttled_write_count = \ table_stats.get_throttled_write_event_count( table_name, lookback_window_start, lookback_period) throttled_by_provisioned_write_percent = \ table_stats.get_throttled_by_provisioned_write_event_percent( table_name, lookback_window_start, lookback_period) throttled_by_consumed_write_percent = \ table_stats.get_throttled_by_consumed_write_percent( table_name, lookback_window_start, lookback_period) writes_upper_threshold = \ get_table_option(key_name, 'writes_upper_threshold') writes_lower_threshold = \ get_table_option(key_name, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_table_option(key_name, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_table_option(key_name, 'increase_writes_unit') increase_writes_with = \ get_table_option(key_name, 'increase_writes_with') decrease_writes_unit = \ get_table_option(key_name, 'decrease_writes_unit') decrease_writes_with = \ get_table_option(key_name, 'decrease_writes_with') min_provisioned_writes = \ get_table_option(key_name, 'min_provisioned_writes') max_provisioned_writes = \ get_table_option(key_name, 'max_provisioned_writes') num_write_checks_before_scale_down = \ get_table_option(key_name, 'num_write_checks_before_scale_down') num_write_checks_reset_percent = \ get_table_option(key_name, 'num_write_checks_reset_percent') increase_throttled_by_provisioned_writes_unit = \ get_table_option( key_name, 'increase_throttled_by_provisioned_writes_unit') increase_throttled_by_provisioned_writes_scale = \ get_table_option( key_name, 'increase_throttled_by_provisioned_writes_scale') increase_throttled_by_consumed_writes_unit = \ get_table_option( key_name, 'increase_throttled_by_consumed_writes_unit') increase_throttled_by_consumed_writes_scale = \ get_table_option( key_name, 'increase_throttled_by_consumed_writes_scale') increase_consumed_writes_unit = \ get_table_option(key_name, 'increase_consumed_writes_unit') increase_consumed_writes_with = \ get_table_option(key_name, 'increase_consumed_writes_with') increase_consumed_writes_scale = \ get_table_option(key_name, 'increase_consumed_writes_scale') decrease_consumed_writes_unit = \ get_table_option(key_name, 'decrease_consumed_writes_unit') decrease_consumed_writes_with = \ get_table_option(key_name, 'decrease_consumed_writes_with') decrease_consumed_writes_scale = \ get_table_option(key_name, 'decrease_consumed_writes_scale') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_write_units = current_write_units # Reset consecutive write count if num_write_checks_reset_percent # is reached if num_write_checks_reset_percent: if consumed_write_units_percent >= num_write_checks_reset_percent: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: Consumed percent {1} is ' 'greater than reset percent: {2}'.format( table_name, consumed_write_units_percent, num_write_checks_reset_percent)) num_consec_write_checks = 0 # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_writes_up_scaling'): logger.debug( '{0} - Up scaling event detected. No action taken as scaling ' 'up writes has been disabled in the configuration'.format( table_name)) else: # If local/granular values not specified use global values increase_consumed_writes_unit = \ increase_consumed_writes_unit or increase_writes_unit increase_throttled_by_provisioned_writes_unit = ( increase_throttled_by_provisioned_writes_unit or increase_writes_unit) increase_throttled_by_consumed_writes_unit = \ increase_throttled_by_consumed_writes_unit or increase_writes_unit increase_consumed_writes_with = \ increase_consumed_writes_with or increase_writes_with # Initialise variables to store calculated provisioning throttled_by_provisioned_calculated_provisioning = scale_reader( increase_throttled_by_provisioned_writes_scale, throttled_by_provisioned_write_percent) throttled_by_consumed_calculated_provisioning = scale_reader( increase_throttled_by_consumed_writes_scale, throttled_by_consumed_write_percent) consumed_calculated_provisioning = scale_reader( increase_consumed_writes_scale, consumed_write_units_percent) throttled_count_calculated_provisioning = 0 calculated_provisioning = 0 # Increase needed due to high throttled to provisioned ratio if throttled_by_provisioned_calculated_provisioning: if increase_throttled_by_provisioned_writes_unit == 'percent': throttled_by_provisioned_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: throttled_by_provisioned_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Increase needed due to high throttled to consumed ratio if throttled_by_consumed_calculated_provisioning: if increase_throttled_by_consumed_writes_unit == 'percent': throttled_by_consumed_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: throttled_by_consumed_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Increase needed due to high CU consumption if consumed_calculated_provisioning: if increase_consumed_writes_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) elif (writes_upper_threshold and consumed_write_units_percent > writes_upper_threshold and not increase_consumed_writes_scale): if increase_consumed_writes_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, increase_consumed_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, increase_consumed_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Increase needed due to high throttling if (throttled_writes_upper_threshold and throttled_write_count > throttled_writes_upper_threshold): if increase_writes_unit == 'percent': throttled_count_calculated_provisioning = \ calculators.increase_writes_in_percent( updated_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: throttled_count_calculated_provisioning = \ calculators.increase_writes_in_units( updated_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Determine which metric requires the most scaling if (throttled_by_provisioned_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_provisioned_calculated_provisioning scale_reason = ( "due to throttled events by provisioned " "units threshold being exceeded") if (throttled_by_consumed_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_consumed_calculated_provisioning scale_reason = ( "due to throttled events by consumed " "units threshold being exceeded") if consumed_calculated_provisioning > calculated_provisioning: calculated_provisioning = consumed_calculated_provisioning scale_reason = "due to consumed threshold being exceeded" if throttled_count_calculated_provisioning > calculated_provisioning: calculated_provisioning = throttled_count_calculated_provisioning scale_reason = "due to throttled events threshold being exceeded" if calculated_provisioning > current_write_units: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up {1}'.format( table_name, scale_reason)) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning # Decrease needed due to low CU consumption if not update_needed: # If local/granular values not specified use global values decrease_consumed_writes_unit = \ decrease_consumed_writes_unit or decrease_writes_unit decrease_consumed_writes_with = \ decrease_consumed_writes_with or decrease_writes_with # Initialise variables to store calculated provisioning consumed_calculated_provisioning = scale_reader_decrease( decrease_consumed_writes_scale, consumed_write_units_percent) calculated_provisioning = None # Exit if down scaling has been disabled if not get_table_option(key_name, 'enable_writes_down_scaling'): logger.debug( '{0} - Down scaling event detected. No action taken as scaling' ' down writes has been disabled in the configuration'.format( table_name)) # Exit if writes == 0% and downscaling has been disabled at 0% elif (consumed_write_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Down scaling event detected. No action taken as scaling' ' down writes is not done when usage is at 0%'.format( table_name)) # Exit if writes are still throttled elif (throttled_writes_upper_threshold and throttled_write_count > throttled_writes_upper_threshold): logger.info( '{0} - Down scaling event detected. No action taken as there' ' are still throttled writes'.format(table_name)) else: if consumed_calculated_provisioning: if decrease_consumed_writes_unit == 'percent': calculated_provisioning = \ calculators.decrease_writes_in_percent( updated_write_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_writes'), table_name) else: calculated_provisioning = \ calculators.decrease_writes_in_units( updated_write_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_writes'), table_name) elif (writes_lower_threshold and consumed_write_units_percent < writes_lower_threshold and not decrease_consumed_writes_scale): if decrease_consumed_writes_unit == 'percent': calculated_provisioning = \ calculators.decrease_writes_in_percent( updated_write_units, decrease_consumed_writes_with, get_table_option( key_name, 'min_provisioned_writes'), table_name) else: calculated_provisioning = \ calculators.decrease_writes_in_units( updated_write_units, decrease_consumed_writes_with, get_table_option( key_name, 'min_provisioned_writes'), table_name) if (calculated_provisioning and current_write_units != calculated_provisioning): num_consec_write_checks += 1 if num_consec_write_checks >= \ num_write_checks_before_scale_down: update_needed = True updated_write_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info( 'Will not increase writes over max-provisioned-writes ' 'limit ({0} writes)'.format(updated_write_units)) # Ensure that we have met the min-provisioning if min_provisioned_writes: if int(min_provisioned_writes) > int(updated_write_units): update_needed = True updated_write_units = int(min_provisioned_writes) logger.info( '{0} - Increasing writes to meet min-provisioned-writes ' 'limit ({1} writes)'.format(table_name, updated_write_units)) if calculators.is_consumed_over_proposed( current_write_units, updated_write_units, consumed_write_units_percent): update_needed = False updated_write_units = current_write_units logger.info( '{0} - Consumed is over proposed write units. Will leave table at ' 'current setting.'.format(table_name)) logger.info('{0} - Consecutive write checks {1}/{2}'.format( table_name, num_consec_write_checks, num_write_checks_before_scale_down)) return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(table_name, key_name, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_table_option(key_name, 'enable_writes_autoscaling'): logger.info( '{0} - Autoscaling of writes has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_write_units(table_name), 0 update_needed = False try: lookback_window_start = get_table_option(key_name, 'lookback_window_start') lookback_period = get_table_option(key_name, 'lookback_period') current_write_units = dynamodb.get_provisioned_table_write_units( table_name) consumed_write_units_percent = \ table_stats.get_consumed_write_units_percent( table_name, lookback_window_start, lookback_period) throttled_write_count = \ table_stats.get_throttled_write_event_count( table_name, lookback_window_start, lookback_period) throttled_by_provisioned_write_percent = \ table_stats.get_throttled_by_provisioned_write_event_percent( table_name, lookback_window_start, lookback_period) throttled_by_consumed_write_percent = \ table_stats.get_throttled_by_consumed_write_percent( table_name, lookback_window_start, lookback_period) writes_upper_threshold = \ get_table_option(key_name, 'writes_upper_threshold') writes_lower_threshold = \ get_table_option(key_name, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_table_option(key_name, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_table_option(key_name, 'increase_writes_unit') increase_writes_with = \ get_table_option(key_name, 'increase_writes_with') decrease_writes_unit = \ get_table_option(key_name, 'decrease_writes_unit') decrease_writes_with = \ get_table_option(key_name, 'decrease_writes_with') min_provisioned_writes = \ get_table_option(key_name, 'min_provisioned_writes') max_provisioned_writes = \ get_table_option(key_name, 'max_provisioned_writes') num_write_checks_before_scale_down = \ get_table_option(key_name, 'num_write_checks_before_scale_down') num_write_checks_reset_percent = \ get_table_option(key_name, 'num_write_checks_reset_percent') increase_throttled_by_provisioned_writes_unit = \ get_table_option( key_name, 'increase_throttled_by_provisioned_writes_unit') increase_throttled_by_provisioned_writes_scale = \ get_table_option( key_name, 'increase_throttled_by_provisioned_writes_scale') increase_throttled_by_consumed_writes_unit = \ get_table_option( key_name, 'increase_throttled_by_consumed_writes_unit') increase_throttled_by_consumed_writes_scale = \ get_table_option( key_name, 'increase_throttled_by_consumed_writes_scale') increase_consumed_writes_unit = \ get_table_option(key_name, 'increase_consumed_writes_unit') increase_consumed_writes_with = \ get_table_option(key_name, 'increase_consumed_writes_with') increase_consumed_writes_scale = \ get_table_option(key_name, 'increase_consumed_writes_scale') decrease_consumed_writes_unit = \ get_table_option(key_name, 'decrease_consumed_writes_unit') decrease_consumed_writes_with = \ get_table_option(key_name, 'decrease_consumed_writes_with') decrease_consumed_writes_scale = \ get_table_option(key_name, 'decrease_consumed_writes_scale') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_write_units = current_write_units # Reset consecutive write count if num_write_checks_reset_percent # is reached if num_write_checks_reset_percent: if consumed_write_units_percent >= num_write_checks_reset_percent: logger.info('{0} - Resetting the number of consecutive ' 'write checks. Reason: Consumed percent {1} is ' 'greater than reset percent: {2}'.format( table_name, consumed_write_units_percent, num_write_checks_reset_percent)) num_consec_write_checks = 0 # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_writes_up_scaling'): logger.debug( '{0} - Up scaling event detected. No action taken as scaling ' 'up writes has been disabled in the configuration'.format( table_name)) else: # If local/granular values not specified use global values increase_consumed_writes_unit = \ increase_consumed_writes_unit or increase_writes_unit increase_throttled_by_provisioned_writes_unit = ( increase_throttled_by_provisioned_writes_unit or increase_writes_unit) increase_throttled_by_consumed_writes_unit = \ increase_throttled_by_consumed_writes_unit or increase_writes_unit increase_consumed_writes_with = \ increase_consumed_writes_with or increase_writes_with # Initialise variables to store calculated provisioning throttled_by_provisioned_calculated_provisioning = scale_reader( increase_throttled_by_provisioned_writes_scale, throttled_by_provisioned_write_percent) throttled_by_consumed_calculated_provisioning = scale_reader( increase_throttled_by_consumed_writes_scale, throttled_by_consumed_write_percent) consumed_calculated_provisioning = scale_reader( increase_consumed_writes_scale, consumed_write_units_percent) throttled_count_calculated_provisioning = 0 calculated_provisioning = 0 # Increase needed due to high throttled to provisioned ratio if throttled_by_provisioned_calculated_provisioning: if increase_throttled_by_provisioned_writes_unit == 'percent': throttled_by_provisioned_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: throttled_by_provisioned_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, throttled_by_provisioned_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Increase needed due to high throttled to consumed ratio if throttled_by_consumed_calculated_provisioning: if increase_throttled_by_consumed_writes_unit == 'percent': throttled_by_consumed_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: throttled_by_consumed_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, throttled_by_consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Increase needed due to high CU consumption if consumed_calculated_provisioning: if increase_consumed_writes_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, consumed_calculated_provisioning, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) elif (writes_upper_threshold and consumed_write_units_percent > writes_upper_threshold and not increase_consumed_writes_scale): if increase_consumed_writes_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, increase_consumed_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: consumed_calculated_provisioning = \ calculators.increase_writes_in_units( current_write_units, increase_consumed_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Increase needed due to high throttling if (throttled_writes_upper_threshold and throttled_write_count > throttled_writes_upper_threshold): if increase_writes_unit == 'percent': throttled_count_calculated_provisioning = \ calculators.increase_writes_in_percent( updated_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: throttled_count_calculated_provisioning = \ calculators.increase_writes_in_units( updated_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) # Determine which metric requires the most scaling if (throttled_by_provisioned_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_provisioned_calculated_provisioning scale_reason = ("due to throttled events by provisioned " "units threshold being exceeded") if (throttled_by_consumed_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_consumed_calculated_provisioning scale_reason = ("due to throttled events by consumed " "units threshold being exceeded") if consumed_calculated_provisioning > calculated_provisioning: calculated_provisioning = consumed_calculated_provisioning scale_reason = "due to consumed threshold being exceeded" if throttled_count_calculated_provisioning > calculated_provisioning: calculated_provisioning = throttled_count_calculated_provisioning scale_reason = "due to throttled events threshold being exceeded" if calculated_provisioning > current_write_units: logger.info('{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up {1}'.format( table_name, scale_reason)) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning # Decrease needed due to low CU consumption if not update_needed: # If local/granular values not specified use global values decrease_consumed_writes_unit = \ decrease_consumed_writes_unit or decrease_writes_unit decrease_consumed_writes_with = \ decrease_consumed_writes_with or decrease_writes_with # Initialise variables to store calculated provisioning consumed_calculated_provisioning = scale_reader_decrease( decrease_consumed_writes_scale, consumed_write_units_percent) calculated_provisioning = None # Exit if down scaling has been disabled if not get_table_option(key_name, 'enable_writes_down_scaling'): logger.debug( '{0} - Down scaling event detected. No action taken as scaling' ' down writes has been disabled in the configuration'.format( table_name)) # Exit if writes == 0% and downscaling has been disabled at 0% elif (consumed_write_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Down scaling event detected. No action taken as scaling' ' down writes is not done when usage is at 0%'.format( table_name)) # Exit if writes are still throttled elif (throttled_writes_upper_threshold and throttled_write_count > throttled_writes_upper_threshold): logger.info( '{0} - Down scaling event detected. No action taken as there' ' are still throttled writes'.format(table_name)) else: if consumed_calculated_provisioning: if decrease_consumed_writes_unit == 'percent': calculated_provisioning = \ calculators.decrease_writes_in_percent( updated_write_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_writes'), table_name) else: calculated_provisioning = \ calculators.decrease_writes_in_units( updated_write_units, consumed_calculated_provisioning, get_table_option( key_name, 'min_provisioned_writes'), table_name) elif (writes_lower_threshold and consumed_write_units_percent < writes_lower_threshold and not decrease_consumed_writes_scale): if decrease_consumed_writes_unit == 'percent': calculated_provisioning = \ calculators.decrease_writes_in_percent( updated_write_units, decrease_consumed_writes_with, get_table_option( key_name, 'min_provisioned_writes'), table_name) else: calculated_provisioning = \ calculators.decrease_writes_in_units( updated_write_units, decrease_consumed_writes_with, get_table_option( key_name, 'min_provisioned_writes'), table_name) if (calculated_provisioning and current_write_units != calculated_provisioning): num_consec_write_checks += 1 if num_consec_write_checks >= \ num_write_checks_before_scale_down: update_needed = True updated_write_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info('Will not increase writes over max-provisioned-writes ' 'limit ({0} writes)'.format(updated_write_units)) # Ensure that we have met the min-provisioning if min_provisioned_writes: if int(min_provisioned_writes) > int(updated_write_units): update_needed = True updated_write_units = int(min_provisioned_writes) logger.info( '{0} - Increasing writes to meet min-provisioned-writes ' 'limit ({1} writes)'.format(table_name, updated_write_units)) if calculators.is_consumed_over_proposed(current_write_units, updated_write_units, consumed_write_units_percent): update_needed = False updated_write_units = current_write_units logger.info( '{0} - Consumed is over proposed write units. Will leave table at ' 'current setting.'.format(table_name)) logger.info('{0} - Consecutive write checks {1}/{2}'.format( table_name, num_consec_write_checks, num_write_checks_before_scale_down)) return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(table_name, key_name, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_table_option(key_name, 'enable_writes_autoscaling'): logger.info( '{0} - Autoscaling of writes has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_write_units(table_name), 0 update_needed = False try: lookback_window_start = get_table_option(key_name, 'lookback_window_start') current_write_units = dynamodb.get_provisioned_table_write_units( table_name) consumed_write_units_percent = \ table_stats.get_consumed_write_units_percent( table_name, lookback_window_start) throttled_write_count = \ table_stats.get_throttled_write_event_count( table_name, lookback_window_start) writes_upper_threshold = \ get_table_option(key_name, 'writes_upper_threshold') writes_lower_threshold = \ get_table_option(key_name, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_table_option(key_name, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_table_option(key_name, 'increase_writes_unit') increase_writes_with = \ get_table_option(key_name, 'increase_writes_with') decrease_writes_unit = \ get_table_option(key_name, 'decrease_writes_unit') decrease_writes_with = \ get_table_option(key_name, 'decrease_writes_with') min_provisioned_writes = \ get_table_option(key_name, 'min_provisioned_writes') max_provisioned_writes = \ get_table_option(key_name, 'max_provisioned_writes') num_write_checks_before_scale_down = \ get_table_option(key_name, 'num_write_checks_before_scale_down') num_write_checks_reset_percent = \ get_table_option(key_name, 'num_write_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_write_units = current_write_units # Reset consecutive write count num_write_checks_reset_percent is reached if num_write_checks_reset_percent: if consumed_write_units_percent >= num_write_checks_reset_percent: logger.info('{0} - Resetting the number of consecutive ' 'write checks. Reason: Consumed percent {1} is ' 'greater than reset percent: {2}'.format( table_name, consumed_write_units_percent, num_write_checks_reset_percent)) num_consec_write_checks = 0 # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Scaling down writes is not done when usage is at 0%'.format( table_name)) # Increase needed due to high CU consumption elif consumed_write_units_percent >= writes_upper_threshold: # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_writes_up_scaling'): logger.debug( '{0} - Up scaling event detected. No action taken as scaling ' 'up writes has been disabled in the configuration'.format( table_name)) else: if increase_writes_unit == 'percent': calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: calculated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) if current_write_units != calculated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning # Increase needed due to high throttling elif throttled_write_count > throttled_writes_upper_threshold: if throttled_writes_upper_threshold > 0: if increase_writes_unit == 'percent': calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: calculated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) if current_write_units != calculated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning # Decrease needed due to low CU consumption elif consumed_write_units_percent <= writes_lower_threshold: # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_writes_down_scaling'): logger.debug( '{0} - Down scaling event detected. No action taken as scaling ' 'down writes has been disabled in the configuration'.format( table_name)) else: if decrease_writes_unit == 'percent': calculated_provisioning = \ calculators.decrease_writes_in_percent( current_write_units, decrease_writes_with, get_table_option(key_name, 'min_provisioned_writes'), table_name) else: calculated_provisioning = calculators.decrease_writes_in_units( current_write_units, decrease_writes_with, get_table_option(key_name, 'min_provisioned_writes'), table_name) if current_write_units != calculated_provisioning: num_consec_write_checks = num_consec_write_checks + 1 if (num_consec_write_checks >= num_write_checks_before_scale_down): update_needed = True updated_write_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info('Will not increase writes over max-provisioned-writes ' 'limit ({0} writes)'.format(updated_write_units)) # Ensure that we have met the min-provisioning if min_provisioned_writes: if int(min_provisioned_writes) > int(updated_write_units): update_needed = True updated_write_units = int(min_provisioned_writes) logger.info( '{0} - Increasing writes to meet min-provisioned-writes ' 'limit ({1} writes)'.format(table_name, updated_write_units)) logger.info('{0} - Consecutive write checks {1}/{2}'.format( table_name, num_consec_write_checks, num_write_checks_before_scale_down)) return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_table_option(key_name, 'enable_writes_autoscaling'): logger.info( '{0} - Autoscaling of writes has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_write_units(table_name), 0 update_needed = False try: lookback_window_start = get_table_option( key_name, 'lookback_window_start') current_write_units = dynamodb.get_provisioned_table_write_units( table_name) consumed_write_units_percent = \ table_stats.get_consumed_write_units_percent( table_name, lookback_window_start) throttled_write_count = \ table_stats.get_throttled_write_event_count( table_name, lookback_window_start) writes_upper_threshold = \ get_table_option(key_name, 'writes_upper_threshold') writes_lower_threshold = \ get_table_option(key_name, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_table_option(key_name, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_table_option(key_name, 'increase_writes_unit') increase_writes_with = \ get_table_option(key_name, 'increase_writes_with') decrease_writes_unit = \ get_table_option(key_name, 'decrease_writes_unit') decrease_writes_with = \ get_table_option(key_name, 'decrease_writes_with') min_provisioned_writes = \ get_table_option(key_name, 'min_provisioned_writes') max_provisioned_writes = \ get_table_option(key_name, 'max_provisioned_writes') num_write_checks_before_scale_down = \ get_table_option(key_name, 'num_write_checks_before_scale_down') num_write_checks_reset_percent = \ get_table_option(key_name, 'num_write_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_write_units = current_write_units # Reset consecutive write count num_write_checks_reset_percent is reached if num_write_checks_reset_percent: if consumed_write_units_percent >= num_write_checks_reset_percent: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: Consumed percent {1} is ' 'greater than reset percent: {2}'.format( table_name, consumed_write_units_percent, num_write_checks_reset_percent)) num_consec_write_checks = 0 # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Scaling down writes is not done when usage is at 0%'.format( table_name)) # Increase needed due to high CU consumption elif consumed_write_units_percent >= writes_upper_threshold: # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_writes_up_scaling'): logger.debug( '{0} - Up scaling event detected. No action taken as scaling ' 'up writes has been disabled in the configuration'.format( table_name)) else: if increase_writes_unit == 'percent': calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: calculated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) if current_write_units != calculated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning # Increase needed due to high throttling elif throttled_write_count > throttled_writes_upper_threshold: if throttled_writes_upper_threshold > 0: if increase_writes_unit == 'percent': calculated_provisioning = \ calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) else: calculated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), consumed_write_units_percent, table_name) if current_write_units != calculated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning # Decrease needed due to low CU consumption elif consumed_write_units_percent <= writes_lower_threshold: # Exit if up scaling has been disabled if not get_table_option(key_name, 'enable_writes_down_scaling'): logger.debug( '{0} - Down scaling event detected. No action taken as scaling ' 'down writes has been disabled in the configuration'.format( table_name)) else: if decrease_writes_unit == 'percent': calculated_provisioning = \ calculators.decrease_writes_in_percent( current_write_units, decrease_writes_with, get_table_option(key_name, 'min_provisioned_writes'), table_name) else: calculated_provisioning = calculators.decrease_writes_in_units( current_write_units, decrease_writes_with, get_table_option(key_name, 'min_provisioned_writes'), table_name) if current_write_units != calculated_provisioning: num_consec_write_checks = num_consec_write_checks + 1 if (num_consec_write_checks >= num_write_checks_before_scale_down): update_needed = True updated_write_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info( 'Will not increase writes over max-provisioned-writes ' 'limit ({0} writes)'.format(updated_write_units)) # Ensure that we have met the min-provisioning if min_provisioned_writes: if int(min_provisioned_writes) > int(updated_write_units): update_needed = True updated_write_units = int(min_provisioned_writes) logger.info( '{0} - Increasing writes to meet min-provisioned-writes ' 'limit ({1} writes)'.format(table_name, updated_write_units)) logger.info('{0} - Consecutive write checks {1}/{2}'.format( table_name, num_consec_write_checks, num_write_checks_before_scale_down)) return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :returns: (bool, int) -- update_needed, updated_write_units """ if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'): logger.info( '{0} - GSI: {1} - ' 'Autoscaling of writes has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) update_needed = False try: updated_write_units = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) consumed_write_units_percent = \ gsi_stats.get_consumed_write_units_percent(table_name, gsi_name) throttled_write_count = \ gsi_stats.get_throttled_write_event_count(table_name, gsi_name) writes_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'writes_upper_threshold') writes_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_gsi_option(table_key, gsi_key, 'increase_writes_unit') increase_writes_with = \ get_gsi_option(table_key, gsi_key, 'increase_writes_with') decrease_writes_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_writes_unit') decrease_writes_with = \ get_gsi_option(table_key, gsi_key, 'decrease_writes_with') max_provisioned_writes = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_writes') except JSONResponseError: raise except BotoServerError: raise # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down writes is not done when usage is at 0%'.format( table_name, gsi_name)) elif consumed_write_units_percent >= writes_upper_threshold: if increase_writes_unit == 'percent': updated_provisioning = calculators.increase_writes_in_percent( updated_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: updated_provisioning = calculators.increase_writes_in_units( updated_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if updated_write_units != updated_provisioning: update_needed = True updated_write_units = updated_provisioning elif throttled_write_count > throttled_writes_upper_threshold: if throttled_writes_upper_threshold > 0: if increase_writes_unit == 'percent': updated_provisioning = calculators.increase_writes_in_percent( updated_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: updated_provisioning = calculators.increase_writes_in_units( updated_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if updated_write_units != updated_provisioning: update_needed = True updated_write_units = updated_provisioning elif consumed_write_units_percent <= writes_lower_threshold: if decrease_writes_unit == 'percent': updated_provisioning = calculators.decrease_writes_in_percent( updated_write_units, decrease_writes_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: updated_provisioning = calculators.decrease_writes_in_units( updated_write_units, decrease_writes_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if updated_write_units != updated_provisioning: update_needed = True updated_write_units = updated_provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info( '{0} - GSI: {1} - ' 'Will not increase writes over gsi-max-provisioned-writes ' 'limit ({2} writes)'.format( table_name, gsi_name, updated_write_units)) return update_needed, int(updated_write_units)
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_gsi_option(table_key, gsi_key, "enable_writes_autoscaling"): logger.info("{0} - GSI: {1} - " "Autoscaling of writes has been disabled".format(table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name), 0 update_needed = False try: current_write_units = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name) consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name) throttled_write_count = gsi_stats.get_throttled_write_event_count(table_name, gsi_name) writes_upper_threshold = get_gsi_option(table_key, gsi_key, "writes_upper_threshold") writes_lower_threshold = get_gsi_option(table_key, gsi_key, "writes_lower_threshold") throttled_writes_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_writes_upper_threshold") increase_writes_unit = get_gsi_option(table_key, gsi_key, "increase_writes_unit") increase_writes_with = get_gsi_option(table_key, gsi_key, "increase_writes_with") decrease_writes_unit = get_gsi_option(table_key, gsi_key, "decrease_writes_unit") decrease_writes_with = get_gsi_option(table_key, gsi_key, "decrease_writes_with") max_provisioned_writes = get_gsi_option(table_key, gsi_key, "max_provisioned_writes") num_write_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_write_checks_before_scale_down") num_write_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_write_checks_reset_percent") except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current write unit value updated_write_units = current_write_units # Reset write consecutive count if num_write_checks_reset_percent is reached if num_write_checks_reset_percent: if consumed_write_units_percent >= num_write_checks_reset_percent: logger.info( "{0} - GSI: {1} - Resetting the number of consecutive " "write checks. Reason: Consumed percent {2} is " "greater than reset percent: {3}".format( table_name, gsi_name, consumed_write_units_percent, num_write_checks_reset_percent ) ) num_consec_write_checks = 0 # Check if we should update write provisioning if consumed_write_units_percent == 0 and not get_gsi_option( table_key, gsi_key, "allow_scaling_down_writes_on_0_percent" ): logger.info( "{0} - GSI: {1} - " "Scaling down writes is not done when usage is at 0%".format(table_name, gsi_name) ) elif consumed_write_units_percent >= writes_upper_threshold: if increase_writes_unit == "percent": calculated_provisioning = calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) else: calculated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) if current_write_units != calculated_provisioning: logger.info( "{0} - GSI: {1} - Resetting the number of consecutive " "write checks. Reason: scale up event detected".format(table_name, gsi_name) ) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning elif throttled_write_count > throttled_writes_upper_threshold: if throttled_writes_upper_threshold > 0: if increase_writes_unit == "percent": calculated_provisioning = calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) else: calculated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) if current_write_units != calculated_provisioning: logger.info( "{0} - GSI: {1} - Resetting the number of consecutive " "write checks. Reason: scale up event detected".format(table_name, gsi_name) ) num_consec_write_checks = 0 update_needed = True updated_write_units = calculated_provisioning elif consumed_write_units_percent <= writes_lower_threshold: if decrease_writes_unit == "percent": calculated_provisioning = calculators.decrease_writes_in_percent( current_write_units, decrease_writes_with, get_gsi_option(table_key, gsi_key, "min_provisioned_writes"), "{0} - GSI: {1}".format(table_name, gsi_name), ) else: calculated_provisioning = calculators.decrease_writes_in_units( current_write_units, decrease_writes_with, get_gsi_option(table_key, gsi_key, "min_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) if current_write_units != calculated_provisioning: num_consec_write_checks = num_consec_write_checks + 1 if num_consec_write_checks >= num_write_checks_before_scale_down: update_needed = True updated_write_units = calculated_provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info( "{0} - GSI: {1} - " "Will not increase writes over gsi-max-provisioned-writes " "limit ({2} writes)".format(table_name, gsi_name, updated_write_units) ) logger.info( "{0} - GSI: {1} - Consecutive write checks {2}/{3}".format( table_name, gsi_name, num_consec_write_checks, num_write_checks_before_scale_down ) ) return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_write_units, num_consec_write_checks """ if not get_table_option(key_name, 'enable_writes_autoscaling'): logger.info( '{0} - Autoscaling of reads has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_write_units(table_name), 0 update_needed = False try: current_write_units = dynamodb.get_provisioned_table_write_units( table_name) consumed_write_units_percent = \ table_stats.get_consumed_write_units_percent(table_name) throttled_write_count = \ table_stats.get_throttled_write_event_count(table_name) writes_upper_threshold = \ get_table_option(key_name, 'writes_upper_threshold') writes_lower_threshold = \ get_table_option(key_name, 'writes_lower_threshold') throttled_writes_upper_threshold = \ get_table_option(key_name, 'throttled_writes_upper_threshold') increase_writes_unit = \ get_table_option(key_name, 'increase_writes_unit') increase_writes_with = \ get_table_option(key_name, 'increase_writes_with') decrease_writes_unit = \ get_table_option(key_name, 'decrease_writes_unit') decrease_writes_with = \ get_table_option(key_name, 'decrease_writes_with') max_provisioned_writes = \ get_table_option(key_name, 'max_provisioned_writes') num_write_checks_before_scale_down = \ get_table_option(key_name, 'num_write_checks_before_scale_down') num_write_checks_reset_percent = \ get_table_option(key_name, 'num_write_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_write_units = current_write_units # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Scaling down writes is not done when usage is at 0%'.format( table_name)) elif consumed_write_units_percent >= writes_upper_threshold: if increase_writes_unit == 'percent': calulated_provisioning = calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), table_name) else: calulated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), table_name) if current_write_units != calulated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calulated_provisioning elif throttled_write_count > throttled_writes_upper_threshold: if throttled_writes_upper_threshold > 0: if increase_writes_unit == 'percent': calulated_provisioning = calculators.increase_writes_in_percent( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), table_name) else: calulated_provisioning = calculators.increase_writes_in_units( current_write_units, increase_writes_with, get_table_option(key_name, 'max_provisioned_writes'), table_name) if current_write_units != calulated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: scale up event detected'.format( table_name)) num_consec_write_checks = 0 update_needed = True updated_write_units = calulated_provisioning elif consumed_write_units_percent >= num_write_checks_reset_percent: logger.info( '{0} - Resetting the number of consecutive ' 'write checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format( table_name, consumed_write_units_percent, num_write_checks_reset_percent)) num_consec_write_checks = 0 elif consumed_write_units_percent <= writes_lower_threshold: if decrease_writes_unit == 'percent': calulated_provisioning = calculators.decrease_writes_in_percent( current_write_units, decrease_writes_with, get_table_option(key_name, 'min_provisioned_writes'), table_name) else: calulated_provisioning = calculators.decrease_writes_in_units( current_write_units, decrease_writes_with, get_table_option(key_name, 'min_provisioned_writes'), table_name) if current_write_units != calulated_provisioning: num_consec_write_checks = num_consec_write_checks + 1 if num_consec_write_checks >= num_write_checks_before_scale_down: update_needed = True updated_write_units = calulated_provisioning if max_provisioned_writes: if int(updated_write_units) > int(max_provisioned_writes): update_needed = True updated_write_units = int(max_provisioned_writes) logger.info( 'Will not increase writes over max-provisioned-writes ' 'limit ({0} writes)'.format(updated_write_units)) logger.info('{0} - Consecutive write checks {1}/{2}'.format( table_name, num_consec_write_checks, num_write_checks_before_scale_down)) return update_needed, updated_write_units, num_consec_write_checks