def test_decrease_reads_in_percent(self):
     """ Ensure that a regular decrease works """
     result = calculators.decrease_reads_in_percent(200, 90, 1, 'test')
     self.assertEqual(result, 20)
Exemple #2
0
def __ensure_provisioning_reads(
        table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of reads has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        current_read_units = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        consumed_read_units_percent = \
            gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
        throttled_read_count = \
            gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
        reads_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
        increase_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
        decrease_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
        increase_reads_with = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_with')
        decrease_reads_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
        throttled_reads_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_reads_upper_threshold')
        max_provisioned_reads = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_read_checks_before_scale_down')
	num_read_checks_reset_percent = \
	    get_gsi_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    if (consumed_read_units_percent == 0 and not
            get_gsi_option(
                table_key,
                gsi_key,
                'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down reads is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_read_units_percent >= reads_upper_threshold:

        if increase_reads_unit == 'percent':
            calulated_provisioning = calculators.increase_reads_in_percent(
                current_read_units,
                increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.increase_reads_in_units(
                current_read_units,
                increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_read_units != calulated_provisioning:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: scale up event detected'.format(
                    table_name))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calulated_provisioning

    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:

            if increase_reads_unit == 'percent':
                calulated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calulated_provisioning = calculators.increase_reads_in_units(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_read_units != calulated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calulated_provisioning
				
    elif consumed_read_units_percent >= num_read_checks_reset_percent:
		logger.info(
            '{0} - Resetting the number of consecutive '
            'read checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format(
                table_name, consumed_read_units_percent, num_read_checks_reset_percent))
		num_consec_read_checks = 0

    elif consumed_read_units_percent <= reads_lower_threshold:

        if decrease_reads_unit == 'percent':
            calulated_provisioning = calculators.decrease_reads_in_percent(
                current_read_units,
                decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.decrease_reads_in_units(
                current_read_units,
                decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_read_units != calulated_provisioning:
            # We need to look at how many times the num_consec_read_checks
            # integer has incremented and Compare to config file value
            num_consec_read_checks = num_consec_read_checks + 1

            if num_consec_read_checks >= num_read_checks_before_scale_down:
                update_needed = True
                updated_read_units = calulated_provisioning

    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over gsi-max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name,
        num_consec_read_checks,
        num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
 def test_decrease_reads_in_percent_hit_min_value(self):
     """ Check that min values are honoured """
     result = calculators.decrease_reads_in_percent(20, 50, 15, 'test')
     self.assertEqual(result, 15)
 def test_decrease_reads_in_percent_more_than_100_percent(self):
     """ Handle decreases of more that 100% """
     result = calculators.decrease_reads_in_percent(20, 120, 1, 'test')
     self.assertEqual(result, 1)
Exemple #5
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(
            key_name, 'lookback_window_start')
        lookback_period = get_table_option(key_name, 'lookback_period')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start, lookback_period)
        throttled_by_provisioned_read_percent = \
            table_stats.get_throttled_by_provisioned_read_event_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_by_consumed_read_percent = \
            table_stats.get_throttled_by_consumed_read_percent(
                table_name, lookback_window_start, lookback_period)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
        increase_throttled_by_provisioned_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_unit')
        increase_throttled_by_provisioned_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_scale')
        increase_throttled_by_consumed_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_unit')
        increase_throttled_by_consumed_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_scale')
        increase_consumed_reads_unit = \
            get_table_option(key_name, 'increase_consumed_reads_unit')
        increase_consumed_reads_with = \
            get_table_option(key_name, 'increase_consumed_reads_with')
        increase_consumed_reads_scale = \
            get_table_option(key_name, 'increase_consumed_reads_scale')
        decrease_consumed_reads_unit = \
            get_table_option(key_name, 'decrease_consumed_reads_unit')
        decrease_consumed_reads_with = \
            get_table_option(key_name, 'decrease_consumed_reads_with')
        decrease_consumed_reads_scale = \
            get_table_option(key_name, 'decrease_consumed_reads_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: Consumed percent {1} is '
                'greater than reset percent: {2}'.format(
                    table_name,
                    consumed_read_units_percent,
                    num_read_checks_reset_percent))

            num_consec_read_checks = 0

    # Exit if up scaling has been disabled
    if not get_table_option(key_name, 'enable_reads_up_scaling'):
        logger.debug(
            '{0} - Up scaling event detected. No action taken as scaling '
            'up reads has been disabled in the configuration'.format(
                table_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_reads_unit = \
            increase_consumed_reads_unit or increase_reads_unit
        increase_throttled_by_provisioned_reads_unit = \
            increase_throttled_by_provisioned_reads_unit or increase_reads_unit
        increase_throttled_by_consumed_reads_unit = \
            increase_throttled_by_consumed_reads_unit or increase_reads_unit

        increase_consumed_reads_with = \
            increase_consumed_reads_with or increase_reads_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_reads_scale,
            throttled_by_provisioned_read_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_reads_scale,
            throttled_by_consumed_read_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_reads_scale,
            consumed_read_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_reads_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_reads_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        elif (reads_upper_threshold
                and consumed_read_units_percent > reads_upper_threshold
                and not increase_consumed_reads_scale):

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttling
        if (throttled_reads_upper_threshold
                and throttled_read_count > throttled_reads_upper_threshold):

            if increase_reads_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        updated_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        updated_read_units,
                        increase_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = (
                "due to throttled events by provisioned "
                "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = (
                "due to throttled events by consumed "
                "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_read_units:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: scale up {1}'.format(
                    table_name, scale_reason))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if not update_needed:
        # If local/granular values not specified use global values
        decrease_consumed_reads_unit = \
            decrease_consumed_reads_unit or decrease_reads_unit

        decrease_consumed_reads_with = \
            decrease_consumed_reads_with or decrease_reads_with

        # Initialise variables to store calculated provisioning
        consumed_calculated_provisioning = scale_reader_decrease(
            decrease_consumed_reads_scale,
            consumed_read_units_percent)
        calculated_provisioning = None

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads has been disabled in the configuration'.format(
                    table_name))
        # Exit if reads == 0% and downscaling has been disabled at 0%
        elif (consumed_read_units_percent == 0 and not
                get_table_option(
                    key_name, 'allow_scaling_down_reads_on_0_percent')):
            logger.info(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads is not done when usage is at 0%'.format(
                    table_name))
        else:
            if consumed_calculated_provisioning:
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
            elif (reads_lower_threshold
                  and consumed_read_units_percent < reads_lower_threshold
                  and not decrease_consumed_reads_scale):
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)

            if (calculated_provisioning
                    and current_read_units != calculated_provisioning):
                num_consec_read_checks += 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info(
                '{0} - Increasing reads to meet min-provisioned-reads '
                'limit ({1} reads)'.format(table_name, updated_read_units))

    if calculators.is_consumed_over_proposed(
            current_read_units,
            updated_read_units,
            consumed_read_units_percent):
        update_needed = False
        updated_read_units = current_read_units
        logger.info(
            '{0} - Consumed is over proposed read units. Will leave table at '
            'current setting.'.format(table_name))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name,
        num_consec_read_checks,
        num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
Exemple #6
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(key_name,
                                                 'lookback_window_start')
        lookback_period = get_table_option(key_name, 'lookback_period')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start, lookback_period)
        throttled_by_provisioned_read_percent = \
            table_stats.get_throttled_by_provisioned_read_event_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_by_consumed_read_percent = \
            table_stats.get_throttled_by_consumed_read_percent(
                table_name, lookback_window_start, lookback_period)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
        increase_throttled_by_provisioned_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_unit')
        increase_throttled_by_provisioned_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_scale')
        increase_throttled_by_consumed_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_unit')
        increase_throttled_by_consumed_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_scale')
        increase_consumed_reads_unit = \
            get_table_option(key_name, 'increase_consumed_reads_unit')
        increase_consumed_reads_with = \
            get_table_option(key_name, 'increase_consumed_reads_with')
        increase_consumed_reads_scale = \
            get_table_option(key_name, 'increase_consumed_reads_scale')
        decrease_consumed_reads_unit = \
            get_table_option(key_name, 'decrease_consumed_reads_unit')
        decrease_consumed_reads_with = \
            get_table_option(key_name, 'decrease_consumed_reads_with')
        decrease_consumed_reads_scale = \
            get_table_option(key_name, 'decrease_consumed_reads_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: Consumed percent {1} is '
                        'greater than reset percent: {2}'.format(
                            table_name, consumed_read_units_percent,
                            num_read_checks_reset_percent))

            num_consec_read_checks = 0

    # Exit if up scaling has been disabled
    if not get_table_option(key_name, 'enable_reads_up_scaling'):
        logger.debug(
            '{0} - Up scaling event detected. No action taken as scaling '
            'up reads has been disabled in the configuration'.format(
                table_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_reads_unit = \
            increase_consumed_reads_unit or increase_reads_unit
        increase_throttled_by_provisioned_reads_unit = \
            increase_throttled_by_provisioned_reads_unit or increase_reads_unit
        increase_throttled_by_consumed_reads_unit = \
            increase_throttled_by_consumed_reads_unit or increase_reads_unit

        increase_consumed_reads_with = \
            increase_consumed_reads_with or increase_reads_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_reads_scale,
            throttled_by_provisioned_read_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_reads_scale,
            throttled_by_consumed_read_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_reads_scale, consumed_read_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_reads_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_reads_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        elif (reads_upper_threshold
              and consumed_read_units_percent > reads_upper_threshold
              and not increase_consumed_reads_scale):

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttling
        if (throttled_reads_upper_threshold
                and throttled_read_count > throttled_reads_upper_threshold):

            if increase_reads_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        updated_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        updated_read_units,
                        increase_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = ("due to throttled events by provisioned "
                            "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = ("due to throttled events by consumed "
                            "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_read_units:
            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: scale up {1}'.format(
                            table_name, scale_reason))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if not update_needed:
        # If local/granular values not specified use global values
        decrease_consumed_reads_unit = \
            decrease_consumed_reads_unit or decrease_reads_unit

        decrease_consumed_reads_with = \
            decrease_consumed_reads_with or decrease_reads_with

        # Initialise variables to store calculated provisioning
        consumed_calculated_provisioning = scale_reader_decrease(
            decrease_consumed_reads_scale, consumed_read_units_percent)
        calculated_provisioning = None

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads has been disabled in the configuration'.format(
                    table_name))
        # Exit if reads == 0% and downscaling has been disabled at 0%
        elif (consumed_read_units_percent == 0 and not get_table_option(
                key_name, 'allow_scaling_down_reads_on_0_percent')):
            logger.info(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads is not done when usage is at 0%'.format(
                    table_name))
        else:
            if consumed_calculated_provisioning:
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
            elif (reads_lower_threshold
                  and consumed_read_units_percent < reads_lower_threshold
                  and not decrease_consumed_reads_scale):
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)

            if (calculated_provisioning
                    and current_read_units != calculated_provisioning):
                num_consec_read_checks += 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info('Will not increase writes over max-provisioned-reads '
                        'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info('{0} - Increasing reads to meet min-provisioned-reads '
                        'limit ({1} reads)'.format(table_name,
                                                   updated_read_units))

    if calculators.is_consumed_over_proposed(current_read_units,
                                             updated_read_units,
                                             consumed_read_units_percent):
        update_needed = False
        updated_read_units = current_read_units
        logger.info(
            '{0} - Consumed is over proposed read units. Will leave table at '
            'current setting.'.format(table_name))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name, num_consec_read_checks, num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
Exemple #7
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(key_name,
                                                 'lookback_window_start')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: Consumed percent {1} is '
                        'greater than reset percent: {2}'.format(
                            table_name, consumed_read_units_percent,
                            num_read_checks_reset_percent))

            num_consec_read_checks = 0

    if (consumed_read_units_percent == 0 and not get_table_option(
            key_name, 'allow_scaling_down_reads_on_0_percent')):
        logger.info(
            '{0} - Scaling down reads is not done when usage is at 0%'.format(
                table_name))

    # Increase needed due to high CU consumption
    elif consumed_read_units_percent >= reads_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_up_scaling'):
            logger.debug(
                '{0} - Up scaling event detected. No action taken as scaling '
                'up reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    updated_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    updated_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_read_units_percent <= reads_lower_threshold:

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                'down reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if decrease_reads_unit == 'percent':
                calculated_provisioning = calculators.decrease_reads_in_percent(
                    updated_read_units, decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)
            else:
                calculated_provisioning = calculators.decrease_reads_in_units(
                    updated_read_units, decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)

            if current_read_units != calculated_provisioning:
                num_consec_read_checks = num_consec_read_checks + 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info('Will not increase writes over max-provisioned-reads '
                        'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info('{0} - Increasing reads to meet min-provisioned-reads '
                        'limit ({1} reads)'.format(table_name,
                                                   updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name, num_consec_read_checks, num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
Exemple #8
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(
            key_name, 'lookback_window_start')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: Consumed percent {1} is '
                'greater than reset percent: {2}'.format(
                    table_name,
                    consumed_read_units_percent,
                    num_read_checks_reset_percent))

            num_consec_read_checks = 0

    if (consumed_read_units_percent == 0 and not
            get_table_option(
                key_name, 'allow_scaling_down_reads_on_0_percent')):
        logger.info(
            '{0} - Scaling down reads is not done when usage is at 0%'.format(
                table_name))

    # Increase needed due to high CU consumption
    elif consumed_read_units_percent >= reads_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_up_scaling'):
            logger.debug(
                '{0} - Up scaling event detected. No action taken as scaling '
                'up reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    updated_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    updated_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_read_units_percent <= reads_lower_threshold:

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                'down reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if decrease_reads_unit == 'percent':
                calculated_provisioning = calculators.decrease_reads_in_percent(
                    updated_read_units,
                    decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)
            else:
                calculated_provisioning = calculators.decrease_reads_in_units(
                    updated_read_units,
                    decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)

            if current_read_units != calculated_provisioning:
                num_consec_read_checks = num_consec_read_checks + 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info(
                '{0} - Increasing reads to meet min-provisioned-reads '
                'limit ({1} reads)'.format(table_name, updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name,
        num_consec_read_checks,
        num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
Exemple #9
0
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :returns: (bool, int) -- update_needed, updated_read_units
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of reads has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)

    update_needed = False
    try:
        updated_read_units = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        consumed_read_units_percent = \
            gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
        throttled_read_count = \
            gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
        reads_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
        increase_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
        decrease_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
        increase_reads_with = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_with')
        decrease_reads_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
        throttled_reads_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_reads_upper_threshold')
        max_provisioned_reads = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    if (consumed_read_units_percent == 0 and not
            get_gsi_option(
                table_key,
                gsi_key,
                'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down reads is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_read_units_percent >= reads_upper_threshold:
        if increase_reads_unit == 'percent':
            updated_provisioning = calculators.increase_reads_in_percent(
                updated_read_units,
                increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            updated_provisioning = calculators.increase_reads_in_units(
                updated_read_units,
                increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:

            if increase_reads_unit == 'percent':
                updated_provisioning = calculators.increase_reads_in_percent(
                    updated_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                updated_provisioning = calculators.increase_reads_in_units(
                    updated_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if updated_read_units != updated_provisioning:
                update_needed = True
                updated_read_units = updated_provisioning

    elif consumed_read_units_percent <= reads_lower_threshold:

        if decrease_reads_unit == 'percent':
            updated_provisioning = calculators.decrease_reads_in_percent(
                updated_read_units,
                decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            updated_provisioning = calculators.decrease_reads_in_units(
                updated_read_units,
                decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over gsi-max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    return update_needed, int(updated_read_units)
Exemple #10
0
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_gsi_option(table_key, gsi_key, "enable_reads_autoscaling"):
        logger.info("{0} - GSI: {1} - " "Autoscaling of reads has been disabled".format(table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name), 0

    update_needed = False
    try:
        current_read_units = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name)
        consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
        throttled_read_count = gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
        reads_upper_threshold = get_gsi_option(table_key, gsi_key, "reads_upper_threshold")
        reads_lower_threshold = get_gsi_option(table_key, gsi_key, "reads_lower_threshold")
        increase_reads_unit = get_gsi_option(table_key, gsi_key, "increase_reads_unit")
        decrease_reads_unit = get_gsi_option(table_key, gsi_key, "decrease_reads_unit")
        increase_reads_with = get_gsi_option(table_key, gsi_key, "increase_reads_with")
        decrease_reads_with = get_gsi_option(table_key, gsi_key, "decrease_reads_with")
        throttled_reads_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_reads_upper_threshold")
        max_provisioned_reads = get_gsi_option(table_key, gsi_key, "max_provisioned_reads")
        num_read_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_read_checks_before_scale_down")
        num_read_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_read_checks_reset_percent")
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info(
                "{0} - GSI: {1} - Resetting the number of consecutive "
                "read checks. Reason: Consumed percent {2} is "
                "greater than reset percent: {3}".format(
                    table_name, gsi_name, consumed_read_units_percent, num_read_checks_reset_percent
                )
            )

            num_consec_read_checks = 0

    if consumed_read_units_percent == 0 and not get_gsi_option(
        table_key, gsi_key, "allow_scaling_down_reads_on_0_percent"
    ):

        logger.info(
            "{0} - GSI: {1} - " "Scaling down reads is not done when usage is at 0%".format(table_name, gsi_name)
        )

    elif consumed_read_units_percent >= reads_upper_threshold:

        if increase_reads_unit == "percent":
            calculated_provisioning = calculators.increase_reads_in_percent(
                current_read_units,
                increase_reads_with,
                get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )
        else:
            calculated_provisioning = calculators.increase_reads_in_units(
                current_read_units,
                increase_reads_with,
                get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )

        if current_read_units != calculated_provisioning:
            logger.info(
                "{0} - Resetting the number of consecutive "
                "read checks. Reason: scale up event detected".format(table_name)
            )
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calculated_provisioning

    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:

            if increase_reads_unit == "percent":
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                    "{0} - GSI: {1}".format(table_name, gsi_name),
                )
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                    "{0} - GSI: {1}".format(table_name, gsi_name),
                )

            if current_read_units != calculated_provisioning:
                logger.info(
                    "{0} - GSI: {1} - Resetting the number of consecutive "
                    "read checks. Reason: scale up event detected".format(table_name, gsi_name)
                )
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    elif consumed_read_units_percent <= reads_lower_threshold:

        if decrease_reads_unit == "percent":
            calculated_provisioning = calculators.decrease_reads_in_percent(
                current_read_units,
                decrease_reads_with,
                get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )
        else:
            calculated_provisioning = calculators.decrease_reads_in_units(
                current_read_units,
                decrease_reads_with,
                get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )

        if current_read_units != calculated_provisioning:
            # We need to look at how many times the num_consec_read_checks
            # integer has incremented and Compare to config file value
            num_consec_read_checks = num_consec_read_checks + 1

            if num_consec_read_checks >= num_read_checks_before_scale_down:
                update_needed = True
                updated_read_units = calculated_provisioning

    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                "Will not increase writes over gsi-max-provisioned-reads "
                "limit ({0} writes)".format(updated_read_units)
            )

    logger.info(
        "{0} - GSI: {1} - Consecutive read checks {2}/{3}".format(
            table_name, gsi_name, num_consec_read_checks, num_read_checks_before_scale_down
        )
    )

    return update_needed, updated_read_units, num_consec_read_checks
Exemple #11
0
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key,
                                num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
        logger.info('{0} - GSI: {1} - '
                    'Autoscaling of reads has been disabled'.format(
                        table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        current_read_units = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        consumed_read_units_percent = \
            gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
        throttled_read_count = \
            gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
        reads_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
        increase_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
        decrease_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
        increase_reads_with = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_with')
        decrease_reads_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
        throttled_reads_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_reads_upper_threshold')
        max_provisioned_reads = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_gsi_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    if (consumed_read_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down reads is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_read_units_percent >= reads_upper_threshold:

        if increase_reads_unit == 'percent':
            calulated_provisioning = calculators.increase_reads_in_percent(
                current_read_units, increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.increase_reads_in_units(
                current_read_units, increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_read_units != calulated_provisioning:
            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: scale up event detected'.format(
                            table_name))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calulated_provisioning

    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:

            if increase_reads_unit == 'percent':
                calulated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units, increase_reads_with,
                    get_gsi_option(table_key, gsi_key,
                                   'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calulated_provisioning = calculators.increase_reads_in_units(
                    current_read_units, increase_reads_with,
                    get_gsi_option(table_key, gsi_key,
                                   'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_read_units != calulated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calulated_provisioning

    elif consumed_read_units_percent >= num_read_checks_reset_percent:
        logger.info(
            '{0} - Resetting the number of consecutive '
            'read checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'
            .format(table_name, consumed_read_units_percent,
                    num_read_checks_reset_percent))
        num_consec_read_checks = 0

    elif consumed_read_units_percent <= reads_lower_threshold:

        if decrease_reads_unit == 'percent':
            calulated_provisioning = calculators.decrease_reads_in_percent(
                current_read_units, decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.decrease_reads_in_units(
                current_read_units, decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_read_units != calulated_provisioning:
            # We need to look at how many times the num_consec_read_checks
            # integer has incremented and Compare to config file value
            num_consec_read_checks = num_consec_read_checks + 1

            if num_consec_read_checks >= num_read_checks_before_scale_down:
                update_needed = True
                updated_read_units = calulated_provisioning

    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over gsi-max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name, num_consec_read_checks, num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks