示例#1
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(key_name,
                                                 'lookback_window_start')
        lookback_period = get_table_option(key_name, 'lookback_period')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start, lookback_period)
        throttled_by_provisioned_read_percent = \
            table_stats.get_throttled_by_provisioned_read_event_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_by_consumed_read_percent = \
            table_stats.get_throttled_by_consumed_read_percent(
                table_name, lookback_window_start, lookback_period)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
        increase_throttled_by_provisioned_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_unit')
        increase_throttled_by_provisioned_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_scale')
        increase_throttled_by_consumed_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_unit')
        increase_throttled_by_consumed_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_scale')
        increase_consumed_reads_unit = \
            get_table_option(key_name, 'increase_consumed_reads_unit')
        increase_consumed_reads_with = \
            get_table_option(key_name, 'increase_consumed_reads_with')
        increase_consumed_reads_scale = \
            get_table_option(key_name, 'increase_consumed_reads_scale')
        decrease_consumed_reads_unit = \
            get_table_option(key_name, 'decrease_consumed_reads_unit')
        decrease_consumed_reads_with = \
            get_table_option(key_name, 'decrease_consumed_reads_with')
        decrease_consumed_reads_scale = \
            get_table_option(key_name, 'decrease_consumed_reads_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: Consumed percent {1} is '
                        'greater than reset percent: {2}'.format(
                            table_name, consumed_read_units_percent,
                            num_read_checks_reset_percent))

            num_consec_read_checks = 0

    # Exit if up scaling has been disabled
    if not get_table_option(key_name, 'enable_reads_up_scaling'):
        logger.debug(
            '{0} - Up scaling event detected. No action taken as scaling '
            'up reads has been disabled in the configuration'.format(
                table_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_reads_unit = \
            increase_consumed_reads_unit or increase_reads_unit
        increase_throttled_by_provisioned_reads_unit = \
            increase_throttled_by_provisioned_reads_unit or increase_reads_unit
        increase_throttled_by_consumed_reads_unit = \
            increase_throttled_by_consumed_reads_unit or increase_reads_unit

        increase_consumed_reads_with = \
            increase_consumed_reads_with or increase_reads_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_reads_scale,
            throttled_by_provisioned_read_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_reads_scale,
            throttled_by_consumed_read_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_reads_scale, consumed_read_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_reads_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_reads_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        elif (reads_upper_threshold
              and consumed_read_units_percent > reads_upper_threshold
              and not increase_consumed_reads_scale):

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttling
        if (throttled_reads_upper_threshold
                and throttled_read_count > throttled_reads_upper_threshold):

            if increase_reads_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        updated_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        updated_read_units,
                        increase_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = ("due to throttled events by provisioned "
                            "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = ("due to throttled events by consumed "
                            "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_read_units:
            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: scale up {1}'.format(
                            table_name, scale_reason))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if not update_needed:
        # If local/granular values not specified use global values
        decrease_consumed_reads_unit = \
            decrease_consumed_reads_unit or decrease_reads_unit

        decrease_consumed_reads_with = \
            decrease_consumed_reads_with or decrease_reads_with

        # Initialise variables to store calculated provisioning
        consumed_calculated_provisioning = scale_reader_decrease(
            decrease_consumed_reads_scale, consumed_read_units_percent)
        calculated_provisioning = None

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads has been disabled in the configuration'.format(
                    table_name))
        # Exit if reads == 0% and downscaling has been disabled at 0%
        elif (consumed_read_units_percent == 0 and not get_table_option(
                key_name, 'allow_scaling_down_reads_on_0_percent')):
            logger.info(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads is not done when usage is at 0%'.format(
                    table_name))
        else:
            if consumed_calculated_provisioning:
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
            elif (reads_lower_threshold
                  and consumed_read_units_percent < reads_lower_threshold
                  and not decrease_consumed_reads_scale):
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)

            if (calculated_provisioning
                    and current_read_units != calculated_provisioning):
                num_consec_read_checks += 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info('Will not increase writes over max-provisioned-reads '
                        'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info('{0} - Increasing reads to meet min-provisioned-reads '
                        'limit ({1} reads)'.format(table_name,
                                                   updated_read_units))

    if calculators.is_consumed_over_proposed(current_read_units,
                                             updated_read_units,
                                             consumed_read_units_percent):
        update_needed = False
        updated_read_units = current_read_units
        logger.info(
            '{0} - Consumed is over proposed read units. Will leave table at '
            'current setting.'.format(table_name))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name, num_consec_read_checks, num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
示例#2
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(
            key_name, 'lookback_window_start')
        lookback_period = get_table_option(key_name, 'lookback_period')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start, lookback_period)
        throttled_by_provisioned_read_percent = \
            table_stats.get_throttled_by_provisioned_read_event_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_by_consumed_read_percent = \
            table_stats.get_throttled_by_consumed_read_percent(
                table_name, lookback_window_start, lookback_period)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
        increase_throttled_by_provisioned_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_unit')
        increase_throttled_by_provisioned_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_reads_scale')
        increase_throttled_by_consumed_reads_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_unit')
        increase_throttled_by_consumed_reads_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_reads_scale')
        increase_consumed_reads_unit = \
            get_table_option(key_name, 'increase_consumed_reads_unit')
        increase_consumed_reads_with = \
            get_table_option(key_name, 'increase_consumed_reads_with')
        increase_consumed_reads_scale = \
            get_table_option(key_name, 'increase_consumed_reads_scale')
        decrease_consumed_reads_unit = \
            get_table_option(key_name, 'decrease_consumed_reads_unit')
        decrease_consumed_reads_with = \
            get_table_option(key_name, 'decrease_consumed_reads_with')
        decrease_consumed_reads_scale = \
            get_table_option(key_name, 'decrease_consumed_reads_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: Consumed percent {1} is '
                'greater than reset percent: {2}'.format(
                    table_name,
                    consumed_read_units_percent,
                    num_read_checks_reset_percent))

            num_consec_read_checks = 0

    # Exit if up scaling has been disabled
    if not get_table_option(key_name, 'enable_reads_up_scaling'):
        logger.debug(
            '{0} - Up scaling event detected. No action taken as scaling '
            'up reads has been disabled in the configuration'.format(
                table_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_reads_unit = \
            increase_consumed_reads_unit or increase_reads_unit
        increase_throttled_by_provisioned_reads_unit = \
            increase_throttled_by_provisioned_reads_unit or increase_reads_unit
        increase_throttled_by_consumed_reads_unit = \
            increase_throttled_by_consumed_reads_unit or increase_reads_unit

        increase_consumed_reads_with = \
            increase_consumed_reads_with or increase_reads_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_reads_scale,
            throttled_by_provisioned_read_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_reads_scale,
            throttled_by_consumed_read_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_reads_scale,
            consumed_read_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_reads_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_reads_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        elif (reads_upper_threshold
                and consumed_read_units_percent > reads_upper_threshold
                and not increase_consumed_reads_scale):

            if increase_consumed_reads_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        current_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Increase needed due to high throttling
        if (throttled_reads_upper_threshold
                and throttled_read_count > throttled_reads_upper_threshold):

            if increase_reads_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_percent(
                        updated_read_units,
                        increase_consumed_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_reads_in_units(
                        updated_read_units,
                        increase_reads_with,
                        get_table_option(key_name, 'max_provisioned_reads'),
                        consumed_read_units_percent,
                        table_name)

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = (
                "due to throttled events by provisioned "
                "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = (
                "due to throttled events by consumed "
                "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_read_units:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: scale up {1}'.format(
                    table_name, scale_reason))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if not update_needed:
        # If local/granular values not specified use global values
        decrease_consumed_reads_unit = \
            decrease_consumed_reads_unit or decrease_reads_unit

        decrease_consumed_reads_with = \
            decrease_consumed_reads_with or decrease_reads_with

        # Initialise variables to store calculated provisioning
        consumed_calculated_provisioning = scale_reader_decrease(
            decrease_consumed_reads_scale,
            consumed_read_units_percent)
        calculated_provisioning = None

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads has been disabled in the configuration'.format(
                    table_name))
        # Exit if reads == 0% and downscaling has been disabled at 0%
        elif (consumed_read_units_percent == 0 and not
                get_table_option(
                    key_name, 'allow_scaling_down_reads_on_0_percent')):
            logger.info(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down reads is not done when usage is at 0%'.format(
                    table_name))
        else:
            if consumed_calculated_provisioning:
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
            elif (reads_lower_threshold
                  and consumed_read_units_percent < reads_lower_threshold
                  and not decrease_consumed_reads_scale):
                if decrease_consumed_reads_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_reads_in_percent(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_reads_in_units(
                            updated_read_units,
                            decrease_consumed_reads_with,
                            get_table_option(
                                key_name, 'min_provisioned_reads'),
                            table_name)

            if (calculated_provisioning
                    and current_read_units != calculated_provisioning):
                num_consec_read_checks += 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info(
                '{0} - Increasing reads to meet min-provisioned-reads '
                'limit ({1} reads)'.format(table_name, updated_read_units))

    if calculators.is_consumed_over_proposed(
            current_read_units,
            updated_read_units,
            consumed_read_units_percent):
        update_needed = False
        updated_read_units = current_read_units
        logger.info(
            '{0} - Consumed is over proposed read units. Will leave table at '
            'current setting.'.format(table_name))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name,
        num_consec_read_checks,
        num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
示例#3
0
def __ensure_provisioning_reads(table_name, key_name):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :returns: (bool, int) -- update_needed, updated_read_units
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name)

    update_needed = False
    try:
        updated_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(table_name)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(table_name)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    if (consumed_read_units_percent == 0 and not
            get_table_option(
                key_name, 'allow_scaling_down_reads_on_0_percent')):
        print('1')
        logger.info(
            '{0} - Scaling down reads is not done when usage is at 0%'.format(
                table_name))

    elif consumed_read_units_percent >= reads_upper_threshold:

        if increase_reads_unit == 'percent':
            updated_provisioning = calculators.increase_reads_in_percent(
                updated_read_units,
                increase_reads_with,
                key_name,
                table_name)
        else:
            updated_provisioning = calculators.increase_reads_in_units(
                updated_read_units,
                increase_reads_with,
                key_name,
                table_name)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:
            if increase_reads_unit == 'percent':
                updated_provisioning = calculators.increase_reads_in_percent(
                    updated_read_units,
                    increase_reads_with,
                    key_name,
                    table_name)
            else:
                updated_provisioning = calculators.increase_reads_in_units(
                    updated_read_units,
                    increase_reads_with,
                    key_name,
                    table_name)

            if updated_read_units != updated_provisioning:
                update_needed = True
                updated_read_units = updated_provisioning

    elif consumed_read_units_percent <= reads_lower_threshold:

        if decrease_reads_unit == 'percent':
            updated_provisioning = calculators.decrease_reads_in_percent(
                updated_read_units,
                decrease_reads_with,
                key_name,
                table_name)
        else:
            updated_provisioning = calculators.decrease_reads_in_units(
                updated_read_units,
                decrease_reads_with,
                key_name,
                table_name)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    if max_provisioned_reads:
        if (int(updated_read_units) > int(max_provisioned_reads)):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    return update_needed, int(updated_read_units)
示例#4
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(key_name,
                                                 'lookback_window_start')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: Consumed percent {1} is '
                        'greater than reset percent: {2}'.format(
                            table_name, consumed_read_units_percent,
                            num_read_checks_reset_percent))

            num_consec_read_checks = 0

    if (consumed_read_units_percent == 0 and not get_table_option(
            key_name, 'allow_scaling_down_reads_on_0_percent')):
        logger.info(
            '{0} - Scaling down reads is not done when usage is at 0%'.format(
                table_name))

    # Increase needed due to high CU consumption
    elif consumed_read_units_percent >= reads_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_up_scaling'):
            logger.debug(
                '{0} - Up scaling event detected. No action taken as scaling '
                'up reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    updated_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    updated_read_units, increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent, table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_read_units_percent <= reads_lower_threshold:

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                'down reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if decrease_reads_unit == 'percent':
                calculated_provisioning = calculators.decrease_reads_in_percent(
                    updated_read_units, decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)
            else:
                calculated_provisioning = calculators.decrease_reads_in_units(
                    updated_read_units, decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)

            if current_read_units != calculated_provisioning:
                num_consec_read_checks = num_consec_read_checks + 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info('Will not increase writes over max-provisioned-reads '
                        'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info('{0} - Increasing reads to meet min-provisioned-reads '
                        'limit ({1} reads)'.format(table_name,
                                                   updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name, num_consec_read_checks, num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
示例#5
0
def __ensure_provisioning_reads(table_name, key_name, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_table_option(key_name, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_read_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(
            key_name, 'lookback_window_start')
        current_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = \
            table_stats.get_consumed_read_units_percent(
                table_name, lookback_window_start)
        throttled_read_count = \
            table_stats.get_throttled_read_event_count(
                table_name, lookback_window_start)
        reads_upper_threshold = \
            get_table_option(key_name, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_table_option(key_name, 'reads_lower_threshold')
        throttled_reads_upper_threshold = \
            get_table_option(key_name, 'throttled_reads_upper_threshold')
        increase_reads_with = \
            get_table_option(key_name, 'increase_reads_with')
        increase_reads_unit = \
            get_table_option(key_name, 'increase_reads_unit')
        decrease_reads_with = \
            get_table_option(key_name, 'decrease_reads_with')
        decrease_reads_unit = \
            get_table_option(key_name, 'decrease_reads_unit')
        min_provisioned_reads = \
            get_table_option(key_name, 'min_provisioned_reads')
        max_provisioned_reads = \
            get_table_option(key_name, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_table_option(key_name, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_table_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info(
                '{0} - Resetting the number of consecutive '
                'read checks. Reason: Consumed percent {1} is '
                'greater than reset percent: {2}'.format(
                    table_name,
                    consumed_read_units_percent,
                    num_read_checks_reset_percent))

            num_consec_read_checks = 0

    if (consumed_read_units_percent == 0 and not
            get_table_option(
                key_name, 'allow_scaling_down_reads_on_0_percent')):
        logger.info(
            '{0} - Scaling down reads is not done when usage is at 0%'.format(
                table_name))

    # Increase needed due to high CU consumption
    elif consumed_read_units_percent >= reads_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_up_scaling'):
            logger.debug(
                '{0} - Up scaling event detected. No action taken as scaling '
                'up reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    updated_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    updated_read_units,
                    increase_reads_with,
                    get_table_option(key_name, 'max_provisioned_reads'),
                    consumed_read_units_percent,
                    table_name)

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_read_units_percent <= reads_lower_threshold:

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                'down reads has been disabled in the configuration'.format(
                    table_name))
        else:
            if decrease_reads_unit == 'percent':
                calculated_provisioning = calculators.decrease_reads_in_percent(
                    updated_read_units,
                    decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)
            else:
                calculated_provisioning = calculators.decrease_reads_in_units(
                    updated_read_units,
                    decrease_reads_with,
                    get_table_option(key_name, 'min_provisioned_reads'),
                    table_name)

            if current_read_units != calculated_provisioning:
                num_consec_read_checks = num_consec_read_checks + 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_reads:
        if int(min_provisioned_reads) > int(updated_read_units):
            update_needed = True
            updated_read_units = int(min_provisioned_reads)
            logger.info(
                '{0} - Increasing reads to meet min-provisioned-reads '
                'limit ({1} reads)'.format(table_name, updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name,
        num_consec_read_checks,
        num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
示例#6
0
def __ensure_provisioning_reads(table_name, key_name):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :returns: (bool, int) -- update_needed, updated_read_units
    """
    update_needed = False
    try:
        updated_read_units = dynamodb.get_provisioned_table_read_units(
            table_name)
        consumed_read_units_percent = table_stats.\
            get_consumed_read_units_percent(table_name)
    except JSONResponseError:
        raise

    throttled_read_count = table_stats.get_throttled_read_event_count(
        table_name)

    if (consumed_read_units_percent == 0 and not
            get_table_option(
                key_name, 'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - Scaling down reads is not done when usage is at 0%'.format(
                table_name))

    elif (consumed_read_units_percent >=
            get_table_option(key_name, 'reads_upper_threshold')):

        if get_table_option(key_name, 'increase_reads_unit') == 'percent':
            updated_provisioning = calculators.increase_reads_in_percent(
                updated_read_units,
                get_table_option(key_name, 'increase_reads_with'),
                key_name,
                table_name)
        else:
            updated_provisioning = calculators.increase_reads_in_units(
                updated_read_units,
                get_table_option(key_name, 'increase_reads_with'),
                key_name,
                table_name)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    elif (throttled_read_count >=
            get_table_option(key_name, 'throttled_reads_upper_threshold')):

        if get_table_option(key_name, 'increase_reads_unit') == 'percent':
            updated_provisioning = calculators.increase_reads_in_percent(
                updated_read_units,
                get_table_option(key_name, 'increase_reads_with'),
                key_name,
                table_name)
        else:
            updated_provisioning = calculators.increase_reads_in_units(
                updated_read_units,
                get_table_option(key_name, 'increase_reads_with'),
                key_name,
                table_name)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    elif (consumed_read_units_percent <=
            get_table_option(key_name, 'reads_lower_threshold')):

        if get_table_option(key_name, 'decrease_reads_unit') == 'percent':
            updated_provisioning = calculators.decrease_reads_in_percent(
                updated_read_units,
                get_table_option(key_name, 'decrease_reads_with'),
                key_name,
                table_name)
        else:
            updated_provisioning = calculators.decrease_reads_in_units(
                updated_read_units,
                get_table_option(key_name, 'decrease_reads_with'),
                key_name,
                table_name)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    if get_table_option(key_name, 'max_provisioned_reads'):
        if (int(updated_read_units) >
                int(get_table_option(key_name, 'max_provisioned_reads'))):
            update_needed = True
            updated_read_units = int(
                get_table_option(key_name, 'max_provisioned_reads'))
            logger.info(
                'Will not increase writes over max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    return update_needed, int(updated_read_units)