def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ try: metrics = __get_aws_metric(table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits') except BotoServerError: raise if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_read_units = 0 try: consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float( dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name)) * 100)) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ try: metrics = __get_aws_metric( table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits') except BotoServerError: raise if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 try: consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float(dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name)) * 100)) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug('{0} - GSI: {1} - GSI status is {2}'.format( table_name, gsi_name, gsi_status)) if gsi_status != 'ACTIVE': logger.warning( '{0} - GSI: {1} - Not performing throughput changes when GSI ' 'status is {2}'.format(table_name, gsi_name, gsi_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - GSI: {1} - No changes to perform'.format( table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug("{0} - GSI: {1} - GSI status is {2}".format(table_name, gsi_name, gsi_status)) if gsi_status != "ACTIVE": logger.warning( "{0} - GSI: {1} - Not performing throughput changes when GSI " "status is {2}".format(table_name, gsi_name, gsi_status) ) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, "always_decrease_rw_together"): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu ) if read_units == current_ru and write_units == current_wu: logger.info("{0} - GSI: {1} - No changes to perform".format(table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def get_throttled_by_provisioned_read_event_percent( table_name, gsi_name, lookback_window_start=15, lookback_period=5): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric( table_name, gsi_name, lookback_window_start, lookback_period, 'ReadThrottleEvents') except BotoServerError: raise if metrics: lookback_seconds = lookback_period * 60 throttled_read_events = ( float(metrics[0]['Sum']) / float(lookback_seconds)) else: throttled_read_events = 0 try: gsi_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) throttled_by_provisioned_read_percent = ( float(throttled_read_events) / float(gsi_read_units) * 100) except JSONResponseError: raise logger.info( '{0} - GSI: {1} - Throttled read percent ' 'by provision: {2:.2f}%'.format( table_name, gsi_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def get_consumed_read_units_percent( table_name, gsi_name, lookback_window_start=15, lookback_period=5): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: float -- Number of consumed reads as a percentage of provisioned reads """ try: metrics = __get_aws_metric( table_name, gsi_name, lookback_window_start, lookback_period, 'ConsumedReadCapacityUnits') except BotoServerError: raise if metrics: lookback_seconds = lookback_period * 60 consumed_read_units = ( float(metrics[0]['Sum']) / float(lookback_seconds)) else: consumed_read_units = 0 try: gsi_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = ( float(consumed_read_units) / float(gsi_read_units) * 100) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:.2f}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_throttled_by_provisioned_read_event_percent(logger, table_name, gsi_name, lookback_window_start=15, lookback_period=5): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric(logger, table_name, gsi_name, lookback_window_start, lookback_period, 'ReadThrottleEvents') except BotoServerError: raise if metrics: lookback_seconds = lookback_period * 60 throttled_read_events = (float(metrics[0]['Sum']) / float(lookback_seconds)) else: throttled_read_events = 0 try: gsi_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) throttled_by_provisioned_read_percent = (float(throttled_read_events) / float(gsi_read_units) * 100) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Throttled read percent ' 'by provision: {2:.2f}%'.format( table_name, gsi_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def get_consumed_read_units_percent(logger, table_name, gsi_name, lookback_window_start=15, lookback_period=5): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: float -- Number of consumed reads as a percentage of provisioned reads """ try: metrics = __get_aws_metric(logger, table_name, gsi_name, lookback_window_start, lookback_period, 'ConsumedReadCapacityUnits') except BotoServerError: raise if metrics: lookback_seconds = lookback_period * 60 consumed_read_units = (float(metrics[0]['Sum']) / float(lookback_seconds)) else: consumed_read_units = 0 try: gsi_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = (float(consumed_read_units) / float(gsi_read_units) * 100) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:.2f}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def __ensure_provisioning_reads( table_name, table_key, gsi_name, gsi_key, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'): logger.info( '{0} - GSI: {1} - ' 'Autoscaling of reads has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name), 0 update_needed = False try: current_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = \ gsi_stats.get_consumed_read_units_percent(table_name, gsi_name) throttled_read_count = \ gsi_stats.get_throttled_read_event_count(table_name, gsi_name) reads_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_upper_threshold') reads_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_lower_threshold') increase_reads_unit = \ get_gsi_option(table_key, gsi_key, 'increase_reads_unit') decrease_reads_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') increase_reads_with = \ get_gsi_option(table_key, gsi_key, 'increase_reads_with') decrease_reads_with = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_with') throttled_reads_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_reads_upper_threshold') max_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_reads') num_read_checks_before_scale_down = \ get_gsi_option( table_key, gsi_key, 'num_read_checks_before_scale_down') num_read_checks_reset_percent = \ get_gsi_option(key_name, 'num_read_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units if (consumed_read_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down reads is not done when usage is at 0%'.format( table_name, gsi_name)) elif consumed_read_units_percent >= reads_upper_threshold: if increase_reads_unit == 'percent': calulated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calulated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calulated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: scale up event detected'.format( table_name)) num_consec_read_checks = 0 update_needed = True updated_read_units = calulated_provisioning elif throttled_read_count > throttled_reads_upper_threshold: if throttled_reads_upper_threshold > 0: if increase_reads_unit == 'percent': calulated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calulated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calulated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: scale up event detected'.format( table_name)) num_consec_read_checks = 0 update_needed = True updated_read_units = calulated_provisioning elif consumed_read_units_percent >= num_read_checks_reset_percent: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format( table_name, consumed_read_units_percent, num_read_checks_reset_percent)) num_consec_read_checks = 0 elif consumed_read_units_percent <= reads_lower_threshold: if decrease_reads_unit == 'percent': calulated_provisioning = calculators.decrease_reads_in_percent( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calulated_provisioning = calculators.decrease_reads_in_units( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calulated_provisioning: # We need to look at how many times the num_consec_read_checks # integer has incremented and Compare to config file value num_consec_read_checks = num_consec_read_checks + 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calulated_provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( 'Will not increase writes over gsi-max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) logger.info('{0} - Consecutive read checks {1}/{2}'.format( table_name, num_consec_read_checks, num_read_checks_before_scale_down)) return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :returns: (bool, int) -- update_needed, updated_read_units """ if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'): logger.info( '{0} - GSI: {1} - ' 'Autoscaling of reads has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) update_needed = False try: updated_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = \ gsi_stats.get_consumed_read_units_percent(table_name, gsi_name) throttled_read_count = \ gsi_stats.get_throttled_read_event_count(table_name, gsi_name) reads_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_upper_threshold') reads_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_lower_threshold') increase_reads_unit = \ get_gsi_option(table_key, gsi_key, 'increase_reads_unit') decrease_reads_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') increase_reads_with = \ get_gsi_option(table_key, gsi_key, 'increase_reads_with') decrease_reads_with = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_with') throttled_reads_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_reads_upper_threshold') max_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_reads') except JSONResponseError: raise except BotoServerError: raise if (consumed_read_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down reads is not done when usage is at 0%'.format( table_name, gsi_name)) elif consumed_read_units_percent >= reads_upper_threshold: if increase_reads_unit == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif throttled_read_count > throttled_reads_upper_threshold: if throttled_reads_upper_threshold > 0: if increase_reads_unit == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif consumed_read_units_percent <= reads_lower_threshold: if decrease_reads_unit == 'percent': updated_provisioning = calculators.decrease_reads_in_percent( updated_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: updated_provisioning = calculators.decrease_reads_in_units( updated_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( 'Will not increase writes over gsi-max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) return update_needed, int(updated_read_units)
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'): logger.info('{0} - GSI: {1} - ' 'Autoscaling of reads has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name), 0 update_needed = False try: lookback_window_start = get_gsi_option(table_key, gsi_key, 'lookback_window_start') current_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = \ gsi_stats.get_consumed_read_units_percent( table_name, gsi_name, lookback_window_start) throttled_read_count = \ gsi_stats.get_throttled_read_event_count( table_name, gsi_name, lookback_window_start) reads_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_upper_threshold') reads_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_lower_threshold') increase_reads_unit = \ get_gsi_option(table_key, gsi_key, 'increase_reads_unit') decrease_reads_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') increase_reads_with = \ get_gsi_option(table_key, gsi_key, 'increase_reads_with') decrease_reads_with = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_with') throttled_reads_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_reads_upper_threshold') max_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_reads') num_read_checks_before_scale_down = \ get_gsi_option( table_key, gsi_key, 'num_read_checks_before_scale_down') num_read_checks_reset_percent = \ get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units # Reset consecutive reads if num_read_checks_reset_percent is reached if num_read_checks_reset_percent: if consumed_read_units_percent >= num_read_checks_reset_percent: logger.info('{0} - GSI: {1} - Resetting the number of consecutive ' 'read checks. Reason: Consumed percent {2} is ' 'greater than reset percent: {3}'.format( table_name, gsi_name, consumed_read_units_percent, num_read_checks_reset_percent)) num_consec_read_checks = 0 if (consumed_read_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down reads is not done when usage is at 0%'.format( table_name, gsi_name)) # Increase needed due to high CU consumption elif consumed_read_units_percent >= reads_upper_threshold: # Exit if up scaling has been disabled if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'): logger.debug( '{0} - GSI: {1} - Up scaling event detected. ' 'No action taken as scaling ' 'up reads has been disabled in the configuration'.format( table_name, gsi_name)) else: if increase_reads_unit == 'percent': calculated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: scale up event detected'.format( table_name)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Increase needed due to high throttling elif throttled_read_count > throttled_reads_upper_threshold: if throttled_reads_upper_threshold > 0: if increase_reads_unit == 'percent': calculated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: logger.info( '{0} - GSI: {1} - Resetting the number of consecutive ' 'read checks. Reason: scale up event detected'.format( table_name, gsi_name)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Decrease needed due to low CU consumption elif consumed_read_units_percent <= reads_lower_threshold: # Exit if down scaling has been disabled if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'): logger.debug( '{0} - GSI: {1} - Down scaling event detected. ' 'No action taken as scaling ' 'down reads has been disabled in the configuration'.format( table_name, gsi_name)) else: if decrease_reads_unit == 'percent': calculated_provisioning = calculators.decrease_reads_in_percent( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.decrease_reads_in_units( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: # We need to look at how many times the num_consec_read_checks # integer has incremented and Compare to config file value num_consec_read_checks = num_consec_read_checks + 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( 'Will not increase writes over gsi-max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format( table_name, gsi_name, num_consec_read_checks, num_read_checks_before_scale_down)) return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_gsi_option(table_key, gsi_key, "enable_reads_autoscaling"): logger.info("{0} - GSI: {1} - " "Autoscaling of reads has been disabled".format(table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name), 0 update_needed = False try: current_read_units = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name) consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name) throttled_read_count = gsi_stats.get_throttled_read_event_count(table_name, gsi_name) reads_upper_threshold = get_gsi_option(table_key, gsi_key, "reads_upper_threshold") reads_lower_threshold = get_gsi_option(table_key, gsi_key, "reads_lower_threshold") increase_reads_unit = get_gsi_option(table_key, gsi_key, "increase_reads_unit") decrease_reads_unit = get_gsi_option(table_key, gsi_key, "decrease_reads_unit") increase_reads_with = get_gsi_option(table_key, gsi_key, "increase_reads_with") decrease_reads_with = get_gsi_option(table_key, gsi_key, "decrease_reads_with") throttled_reads_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_reads_upper_threshold") max_provisioned_reads = get_gsi_option(table_key, gsi_key, "max_provisioned_reads") num_read_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_read_checks_before_scale_down") num_read_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_read_checks_reset_percent") except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units # Reset consecutive reads if num_read_checks_reset_percent is reached if num_read_checks_reset_percent: if consumed_read_units_percent >= num_read_checks_reset_percent: logger.info( "{0} - GSI: {1} - Resetting the number of consecutive " "read checks. Reason: Consumed percent {2} is " "greater than reset percent: {3}".format( table_name, gsi_name, consumed_read_units_percent, num_read_checks_reset_percent ) ) num_consec_read_checks = 0 if consumed_read_units_percent == 0 and not get_gsi_option( table_key, gsi_key, "allow_scaling_down_reads_on_0_percent" ): logger.info( "{0} - GSI: {1} - " "Scaling down reads is not done when usage is at 0%".format(table_name, gsi_name) ) elif consumed_read_units_percent >= reads_upper_threshold: if increase_reads_unit == "percent": calculated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) else: calculated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) if current_read_units != calculated_provisioning: logger.info( "{0} - Resetting the number of consecutive " "read checks. Reason: scale up event detected".format(table_name) ) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning elif throttled_read_count > throttled_reads_upper_threshold: if throttled_reads_upper_threshold > 0: if increase_reads_unit == "percent": calculated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) else: calculated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, "max_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) if current_read_units != calculated_provisioning: logger.info( "{0} - GSI: {1} - Resetting the number of consecutive " "read checks. Reason: scale up event detected".format(table_name, gsi_name) ) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning elif consumed_read_units_percent <= reads_lower_threshold: if decrease_reads_unit == "percent": calculated_provisioning = calculators.decrease_reads_in_percent( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, "min_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) else: calculated_provisioning = calculators.decrease_reads_in_units( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, "min_provisioned_reads"), "{0} - GSI: {1}".format(table_name, gsi_name), ) if current_read_units != calculated_provisioning: # We need to look at how many times the num_consec_read_checks # integer has incremented and Compare to config file value num_consec_read_checks = num_consec_read_checks + 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calculated_provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( "Will not increase writes over gsi-max-provisioned-reads " "limit ({0} writes)".format(updated_read_units) ) logger.info( "{0} - GSI: {1} - Consecutive read checks {2}/{3}".format( table_name, gsi_name, num_consec_read_checks, num_read_checks_before_scale_down ) ) return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads( table_name, table_key, gsi_name, gsi_key, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'): logger.info( '{0} - GSI: {1} - ' 'Autoscaling of reads has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name), 0 update_needed = False try: lookback_window_start = get_gsi_option( table_key, gsi_key, 'lookback_window_start') current_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = \ gsi_stats.get_consumed_read_units_percent( table_name, gsi_name, lookback_window_start) throttled_read_count = \ gsi_stats.get_throttled_read_event_count( table_name, gsi_name, lookback_window_start) reads_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_upper_threshold') reads_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_lower_threshold') increase_reads_unit = \ get_gsi_option(table_key, gsi_key, 'increase_reads_unit') decrease_reads_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') increase_reads_with = \ get_gsi_option(table_key, gsi_key, 'increase_reads_with') decrease_reads_with = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_with') throttled_reads_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_reads_upper_threshold') min_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'min_provisioned_reads') max_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_reads') num_read_checks_before_scale_down = \ get_gsi_option( table_key, gsi_key, 'num_read_checks_before_scale_down') num_read_checks_reset_percent = \ get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units # Reset consecutive reads if num_read_checks_reset_percent is reached if num_read_checks_reset_percent: if consumed_read_units_percent >= num_read_checks_reset_percent: logger.info( '{0} - GSI: {1} - Resetting the number of consecutive ' 'read checks. Reason: Consumed percent {2} is ' 'greater than reset percent: {3}'.format( table_name, gsi_name, consumed_read_units_percent, num_read_checks_reset_percent)) num_consec_read_checks = 0 if (consumed_read_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down reads is not done when usage is at 0%'.format( table_name, gsi_name)) # Increase needed due to high CU consumption elif consumed_read_units_percent >= reads_upper_threshold: # Exit if up scaling has been disabled if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'): logger.debug( '{0} - GSI: {1} - Up scaling event detected. ' 'No action taken as scaling ' 'up reads has been disabled in the configuration'.format( table_name, gsi_name)) else: if increase_reads_unit == 'percent': calculated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: logger.info( '{0} - Resetting the number of consecutive ' 'read checks. Reason: scale up event detected'.format( table_name)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Increase needed due to high throttling elif throttled_read_count > throttled_reads_upper_threshold: if throttled_reads_upper_threshold > 0: if increase_reads_unit == 'percent': calculated_provisioning = calculators.increase_reads_in_percent( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.increase_reads_in_units( current_read_units, increase_reads_with, get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: logger.info( '{0} - GSI: {1} - Resetting the number of consecutive ' 'read checks. Reason: scale up event detected'.format( table_name, gsi_name)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Decrease needed due to low CU consumption elif consumed_read_units_percent <= reads_lower_threshold: # Exit if down scaling has been disabled if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'): logger.debug( '{0} - GSI: {1} - Down scaling event detected. ' 'No action taken as scaling ' 'down reads has been disabled in the configuration'.format( table_name, gsi_name)) else: if decrease_reads_unit == 'percent': calculated_provisioning = calculators.decrease_reads_in_percent( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.decrease_reads_in_units( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: # We need to look at how many times the num_consec_read_checks # integer has incremented and Compare to config file value num_consec_read_checks = num_consec_read_checks + 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( '{0} - GSI: {1} - Will not increase writes over ' 'gsi-max-provisioned-reads ' 'limit ({2} writes)'.format( table_name, gsi_name, updated_read_units)) # Ensure that we have met the min-provisioning if min_provisioned_reads: if int(min_provisioned_reads) > int(updated_read_units): update_needed = True updated_read_units = int(min_provisioned_reads) logger.info( '{0} - GSI: {1} - Increasing reads to' 'meet gsi-min-provisioned-reads ' 'limit ({2} reads)'.format( table_name, gsi_name, updated_read_units)) logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format( table_name, gsi_name, num_consec_read_checks, num_read_checks_before_scale_down)) return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads( table_name, table_key, gsi_name, gsi_key, num_consec_read_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :returns: (bool, int, int) update_needed, updated_read_units, num_consec_read_checks """ if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'): logger.info( '{0} - GSI: {1} - ' 'Autoscaling of reads has been disabled'.format( table_name, gsi_name)) return False, dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name), 0 update_needed = False try: lookback_window_start = get_gsi_option( table_key, gsi_key, 'lookback_window_start') current_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) consumed_read_units_percent = \ gsi_stats.get_consumed_read_units_percent( table_name, gsi_name, lookback_window_start) throttled_read_count = \ gsi_stats.get_throttled_read_event_count( table_name, gsi_name, lookback_window_start) throttled_by_provisioned_read_percent = \ gsi_stats.get_throttled_by_provisioned_read_event_percent( table_name, gsi_name, lookback_window_start) throttled_by_consumed_read_percent = \ gsi_stats.get_throttled_by_consumed_read_percent( table_name, gsi_name, lookback_window_start) reads_upper_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_upper_threshold') reads_lower_threshold = \ get_gsi_option(table_key, gsi_key, 'reads_lower_threshold') increase_reads_unit = \ get_gsi_option(table_key, gsi_key, 'increase_reads_unit') decrease_reads_unit = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') increase_reads_with = \ get_gsi_option(table_key, gsi_key, 'increase_reads_with') decrease_reads_with = \ get_gsi_option(table_key, gsi_key, 'decrease_reads_with') throttled_reads_upper_threshold = \ get_gsi_option( table_key, gsi_key, 'throttled_reads_upper_threshold') min_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'min_provisioned_reads') max_provisioned_reads = \ get_gsi_option(table_key, gsi_key, 'max_provisioned_reads') num_read_checks_before_scale_down = \ get_gsi_option( table_key, gsi_key, 'num_read_checks_before_scale_down') num_read_checks_reset_percent = \ get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent') increase_throttled_by_provisioned_reads_unit = \ get_gsi_option( table_key, gsi_key, 'increase_throttled_by_provisioned_reads_unit') increase_throttled_by_provisioned_reads_scale = \ get_gsi_option( table_key, gsi_key, 'increase_throttled_by_provisioned_reads_scale') increase_throttled_by_consumed_reads_unit = \ get_gsi_option( table_key, gsi_key, 'increase_throttled_by_consumed_reads_unit') increase_throttled_by_consumed_reads_scale = \ get_gsi_option( table_key, gsi_key, 'increase_throttled_by_consumed_reads_scale') increase_consumed_reads_unit = \ get_gsi_option(table_key, gsi_key, 'increase_consumed_reads_unit') increase_consumed_reads_with = \ get_gsi_option(table_key, gsi_key, 'increase_consumed_reads_with') increase_consumed_reads_scale = \ get_gsi_option(table_key, gsi_key, 'increase_consumed_reads_scale') except JSONResponseError: raise except BotoServerError: raise # Set the updated units to the current read unit value updated_read_units = current_read_units # Reset consecutive reads if num_read_checks_reset_percent is reached if num_read_checks_reset_percent: if consumed_read_units_percent >= num_read_checks_reset_percent: logger.info( '{0} - GSI: {1} - Resetting the number of consecutive ' 'read checks. Reason: Consumed percent {2} is ' 'greater than reset percent: {3}'.format( table_name, gsi_name, consumed_read_units_percent, num_read_checks_reset_percent)) num_consec_read_checks = 0 if (consumed_read_units_percent == 0 and not get_gsi_option( table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - GSI: {1} - ' 'Scaling down reads is not done when usage is at 0%'.format( table_name, gsi_name)) # Exit if up scaling has been disabled if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'): logger.debug( '{0} - GSI: {1} - Up scaling event detected. No action taken as ' 'scaling up reads has been disabled in the configuration'.format( table_name, gsi_name)) else: # If local/granular values not specified use global values increase_consumed_reads_unit = \ increase_consumed_reads_unit or increase_reads_unit increase_throttled_by_provisioned_reads_unit = \ increase_throttled_by_provisioned_reads_unit or increase_reads_unit increase_throttled_by_consumed_reads_unit = \ increase_throttled_by_consumed_reads_unit or increase_reads_unit increase_consumed_reads_with = \ increase_consumed_reads_with or increase_reads_with # Initialise variables to store calculated provisioning throttled_by_provisioned_calculated_provisioning = scale_reader( increase_throttled_by_provisioned_reads_scale, throttled_by_provisioned_read_percent) throttled_by_consumed_calculated_provisioning = scale_reader( increase_throttled_by_consumed_reads_scale, throttled_by_consumed_read_percent) consumed_calculated_provisioning = scale_reader( increase_consumed_reads_scale, consumed_read_units_percent) throttled_count_calculated_provisioning = 0 calculated_provisioning = 0 # Increase needed due to high throttled to provisioned ratio if throttled_by_provisioned_calculated_provisioning: if increase_throttled_by_provisioned_reads_unit == 'percent': throttled_by_provisioned_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, throttled_by_provisioned_calculated_provisioning, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: throttled_by_provisioned_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, throttled_by_provisioned_calculated_provisioning, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) # Increase needed due to high throttled to consumed ratio if throttled_by_consumed_calculated_provisioning: if increase_throttled_by_consumed_reads_unit == 'percent': throttled_by_consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, throttled_by_consumed_calculated_provisioning, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: throttled_by_consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, throttled_by_consumed_calculated_provisioning, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) # Increase needed due to high CU consumption if consumed_calculated_provisioning: if increase_consumed_reads_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, consumed_calculated_provisioning, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, consumed_calculated_provisioning, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) elif (reads_upper_threshold and consumed_read_units_percent > reads_upper_threshold and not increase_consumed_reads_scale): if increase_consumed_reads_unit == 'percent': consumed_calculated_provisioning = \ calculators.increase_reads_in_percent( current_read_units, increase_consumed_reads_with, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: consumed_calculated_provisioning = \ calculators.increase_reads_in_units( current_read_units, increase_consumed_reads_with, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) # Increase needed due to high throttling if (throttled_reads_upper_threshold and throttled_read_count > throttled_reads_upper_threshold): if increase_reads_unit == 'percent': throttled_count_calculated_provisioning = \ calculators.increase_reads_in_percent( updated_read_units, increase_reads_with, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) else: throttled_count_calculated_provisioning = \ calculators.increase_reads_in_units( updated_read_units, increase_reads_with, get_gsi_option( table_key, gsi_key, 'max_provisioned_reads'), consumed_read_units_percent, '{0} - GSI: {1}'.format(table_name, gsi_name)) # Determine which metric requires the most scaling if (throttled_by_provisioned_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_provisioned_calculated_provisioning scale_reason = ( "due to throttled events by provisioned " "units threshold being exceeded") if (throttled_by_consumed_calculated_provisioning > calculated_provisioning): calculated_provisioning = \ throttled_by_consumed_calculated_provisioning scale_reason = ( "due to throttled events by consumed " "units threshold being exceeded") if consumed_calculated_provisioning > calculated_provisioning: calculated_provisioning = consumed_calculated_provisioning scale_reason = "due to consumed threshold being exceeded" if throttled_count_calculated_provisioning > calculated_provisioning: calculated_provisioning = throttled_count_calculated_provisioning scale_reason = "due to throttled events threshold being exceeded" if calculated_provisioning > current_read_units: logger.info( '{0} - GSI: {1} - Resetting the number of consecutive ' 'read checks. Reason: scale up {2}'.format( table_name, gsi_name, scale_reason)) num_consec_read_checks = 0 update_needed = True updated_read_units = calculated_provisioning # Decrease needed due to low CU consumption if (consumed_read_units_percent <= reads_lower_threshold and not update_needed): # Exit if down scaling has been disabled if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'): logger.debug( '{0} - GSI: {1} - Down scaling event detected. ' 'No action taken as scaling ' 'down reads has been disabled in the configuration'.format( table_name, gsi_name)) else: if decrease_reads_unit == 'percent': calculated_provisioning = calculators.decrease_reads_in_percent( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) else: calculated_provisioning = calculators.decrease_reads_in_units( current_read_units, decrease_reads_with, get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'), '{0} - GSI: {1}'.format(table_name, gsi_name)) if current_read_units != calculated_provisioning: # We need to look at how many times the num_consec_read_checks # integer has incremented and Compare to config file value num_consec_read_checks = num_consec_read_checks + 1 if num_consec_read_checks >= num_read_checks_before_scale_down: update_needed = True updated_read_units = calculated_provisioning # Never go over the configured max provisioning if max_provisioned_reads: if int(updated_read_units) > int(max_provisioned_reads): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( '{0} - GSI: {1} - Will not increase writes over ' 'gsi-max-provisioned-reads ' 'limit ({2} writes)'.format( table_name, gsi_name, updated_read_units)) # Ensure that we have met the min-provisioning if min_provisioned_reads: if int(min_provisioned_reads) > int(updated_read_units): update_needed = True updated_read_units = int(min_provisioned_reads) logger.info( '{0} - GSI: {1} - Increasing reads to' 'meet gsi-min-provisioned-reads ' 'limit ({2} reads)'.format( table_name, gsi_name, updated_read_units)) logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format( table_name, gsi_name, num_consec_read_checks, num_read_checks_before_scale_down)) return update_needed, updated_read_units, num_consec_read_checks