Esempio n. 1
0
def __ensure_provisioning_writes(
        table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of writes has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        current_write_units = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
        consumed_write_units_percent = \
            gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
        throttled_write_count = \
            gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
        writes_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
        increase_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_with')
        decrease_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
        decrease_writes_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
        max_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_write_checks_before_scale_down')
	num_write_checks_reset_percent = \
	    get_gsi_option(key_name, 'num_write_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current write unit value
    updated_write_units = current_write_units

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down writes is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == 'percent':
            calulated_provisioning = calculators.increase_writes_in_percent(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.increase_writes_in_units(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_write_units != calulated_provisioning:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'write checks. Reason: scale up event detected'.format(
                    table_name))
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calulated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                calulated_provisioning = calculators.increase_writes_in_percent(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calulated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_write_units != calulated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calulated_provisioning
				
    elif consumed_write_units_percent >= num_write_checks_reset_percent:
		logger.info(
            '{0} - Resetting the number of consecutive '
            'write checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format(
                table_name, consumed_write_units_percent, num_write_checks_reset_percent))
		num_consec_write_checks = 0

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == 'percent':
            calulated_provisioning = calculators.decrease_writes_in_percent(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.decrease_writes_in_units(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_write_units != calulated_provisioning:
            num_consec_write_checks = num_consec_write_checks + 1

            if num_consec_write_checks >= num_write_checks_before_scale_down:
                update_needed = True
                updated_write_units = calulated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - '
                'Will not increase writes over gsi-max-provisioned-writes '
                'limit ({2} writes)'.format(
                    table_name,
                    gsi_name,
                    updated_write_units))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name,
        num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
 def test_decrease_writes_in_units(self):
     """ Ensure that a regular decrease works """
     result = calculators.decrease_writes_in_units(200, 90, 1, 'test')
     self.assertEqual(result, 110)
 def test_decrease_writes_in_units_hit_min_value(self):
     """ Check that min values are honoured """
     result = calculators.decrease_writes_in_units(20, 50, 15, 'test')
     self.assertEqual(result, 15)
Esempio n. 4
0
def __ensure_provisioning_writes(
        table_name, key_name, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_table_option(key_name, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - Autoscaling of writes has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_write_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(
            key_name, 'lookback_window_start')
        lookback_period = get_table_option(key_name, 'lookback_period')
        current_write_units = dynamodb.get_provisioned_table_write_units(
            table_name)
        consumed_write_units_percent = \
            table_stats.get_consumed_write_units_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_write_count = \
            table_stats.get_throttled_write_event_count(
                table_name, lookback_window_start, lookback_period)
        throttled_by_provisioned_write_percent = \
            table_stats.get_throttled_by_provisioned_write_event_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_by_consumed_write_percent = \
            table_stats.get_throttled_by_consumed_write_percent(
                table_name, lookback_window_start, lookback_period)
        writes_upper_threshold = \
            get_table_option(key_name, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_table_option(key_name, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_table_option(key_name, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_table_option(key_name, 'increase_writes_unit')
        increase_writes_with = \
            get_table_option(key_name, 'increase_writes_with')
        decrease_writes_unit = \
            get_table_option(key_name, 'decrease_writes_unit')
        decrease_writes_with = \
            get_table_option(key_name, 'decrease_writes_with')
        min_provisioned_writes = \
            get_table_option(key_name, 'min_provisioned_writes')
        max_provisioned_writes = \
            get_table_option(key_name, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_table_option(key_name, 'num_write_checks_before_scale_down')
        num_write_checks_reset_percent = \
            get_table_option(key_name, 'num_write_checks_reset_percent')
        increase_throttled_by_provisioned_writes_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_writes_unit')
        increase_throttled_by_provisioned_writes_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_writes_scale')
        increase_throttled_by_consumed_writes_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_writes_unit')
        increase_throttled_by_consumed_writes_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_writes_scale')
        increase_consumed_writes_unit = \
            get_table_option(key_name, 'increase_consumed_writes_unit')
        increase_consumed_writes_with = \
            get_table_option(key_name, 'increase_consumed_writes_with')
        increase_consumed_writes_scale = \
            get_table_option(key_name, 'increase_consumed_writes_scale')
        decrease_consumed_writes_unit = \
            get_table_option(key_name, 'decrease_consumed_writes_unit')
        decrease_consumed_writes_with = \
            get_table_option(key_name, 'decrease_consumed_writes_with')
        decrease_consumed_writes_scale = \
            get_table_option(key_name, 'decrease_consumed_writes_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_write_units = current_write_units

    # Reset consecutive write count if num_write_checks_reset_percent
    # is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info(
                '{0} - Resetting the number of consecutive '
                'write checks. Reason: Consumed percent {1} is '
                'greater than reset percent: {2}'.format(
                    table_name,
                    consumed_write_units_percent,
                    num_write_checks_reset_percent))

            num_consec_write_checks = 0

    # Exit if up scaling has been disabled
    if not get_table_option(key_name, 'enable_writes_up_scaling'):
        logger.debug(
            '{0} - Up scaling event detected. No action taken as scaling '
            'up writes has been disabled in the configuration'.format(
                table_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_writes_unit = \
            increase_consumed_writes_unit or increase_writes_unit
        increase_throttled_by_provisioned_writes_unit = (
            increase_throttled_by_provisioned_writes_unit
            or increase_writes_unit)
        increase_throttled_by_consumed_writes_unit = \
            increase_throttled_by_consumed_writes_unit or increase_writes_unit

        increase_consumed_writes_with = \
            increase_consumed_writes_with or increase_writes_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_writes_scale,
            throttled_by_provisioned_write_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_writes_scale,
            throttled_by_consumed_write_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_writes_scale, consumed_write_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_writes_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_writes_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_writes_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        elif (writes_upper_threshold
              and consumed_write_units_percent > writes_upper_threshold
              and not increase_consumed_writes_scale):

            if increase_consumed_writes_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_consumed_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        increase_consumed_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Increase needed due to high throttling
        if (throttled_writes_upper_threshold and throttled_write_count >
                throttled_writes_upper_threshold):

            if increase_writes_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        updated_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        updated_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = (
                "due to throttled events by provisioned "
                "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = (
                "due to throttled events by consumed "
                "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_write_units:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'write checks. Reason: scale up {1}'.format(
                    table_name, scale_reason))
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if not update_needed:
        # If local/granular values not specified use global values
        decrease_consumed_writes_unit = \
            decrease_consumed_writes_unit or decrease_writes_unit

        decrease_consumed_writes_with = \
            decrease_consumed_writes_with or decrease_writes_with

        # Initialise variables to store calculated provisioning
        consumed_calculated_provisioning = scale_reader_decrease(
            decrease_consumed_writes_scale,
            consumed_write_units_percent)
        calculated_provisioning = None

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_writes_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down writes has been disabled in the configuration'.format(
                    table_name))
        # Exit if writes == 0% and downscaling has been disabled at 0%
        elif (consumed_write_units_percent == 0 and not
                get_table_option(
                    key_name, 'allow_scaling_down_writes_on_0_percent')):
            logger.info(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down writes is not done when usage is at 0%'.format(
                    table_name))
        # Exit if writes are still throttled
        elif (throttled_writes_upper_threshold
              and throttled_write_count > throttled_writes_upper_threshold):
            logger.info(
                '{0} - Down scaling event detected. No action taken as there'
                ' are still throttled writes'.format(table_name))
        else:
            if consumed_calculated_provisioning:
                if decrease_consumed_writes_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_writes_in_percent(
                            updated_write_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_writes_in_units(
                            updated_write_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)
            elif (writes_lower_threshold
                  and consumed_write_units_percent < writes_lower_threshold
                  and not decrease_consumed_writes_scale):
                if decrease_consumed_writes_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_writes_in_percent(
                            updated_write_units,
                            decrease_consumed_writes_with,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_writes_in_units(
                            updated_write_units,
                            decrease_consumed_writes_with,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)

            if (calculated_provisioning and
                    current_write_units != calculated_provisioning):
                num_consec_write_checks += 1

                if num_consec_write_checks >= \
                        num_write_checks_before_scale_down:
                    update_needed = True
                    updated_write_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                'Will not increase writes over max-provisioned-writes '
                'limit ({0} writes)'.format(updated_write_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_writes:
        if int(min_provisioned_writes) > int(updated_write_units):
            update_needed = True
            updated_write_units = int(min_provisioned_writes)
            logger.info(
                '{0} - Increasing writes to meet min-provisioned-writes '
                'limit ({1} writes)'.format(table_name, updated_write_units))

    if calculators.is_consumed_over_proposed(
            current_write_units,
            updated_write_units,
            consumed_write_units_percent):
        update_needed = False
        updated_write_units = current_write_units
        logger.info(
            '{0} - Consumed is over proposed write units. Will leave table at '
            'current setting.'.format(table_name))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name,
        num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
Esempio n. 5
0
def __ensure_provisioning_writes(table_name, key_name,
                                 num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_table_option(key_name, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - Autoscaling of writes has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_write_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(key_name,
                                                 'lookback_window_start')
        lookback_period = get_table_option(key_name, 'lookback_period')
        current_write_units = dynamodb.get_provisioned_table_write_units(
            table_name)
        consumed_write_units_percent = \
            table_stats.get_consumed_write_units_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_write_count = \
            table_stats.get_throttled_write_event_count(
                table_name, lookback_window_start, lookback_period)
        throttled_by_provisioned_write_percent = \
            table_stats.get_throttled_by_provisioned_write_event_percent(
                table_name, lookback_window_start, lookback_period)
        throttled_by_consumed_write_percent = \
            table_stats.get_throttled_by_consumed_write_percent(
                table_name, lookback_window_start, lookback_period)
        writes_upper_threshold = \
            get_table_option(key_name, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_table_option(key_name, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_table_option(key_name, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_table_option(key_name, 'increase_writes_unit')
        increase_writes_with = \
            get_table_option(key_name, 'increase_writes_with')
        decrease_writes_unit = \
            get_table_option(key_name, 'decrease_writes_unit')
        decrease_writes_with = \
            get_table_option(key_name, 'decrease_writes_with')
        min_provisioned_writes = \
            get_table_option(key_name, 'min_provisioned_writes')
        max_provisioned_writes = \
            get_table_option(key_name, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_table_option(key_name, 'num_write_checks_before_scale_down')
        num_write_checks_reset_percent = \
            get_table_option(key_name, 'num_write_checks_reset_percent')
        increase_throttled_by_provisioned_writes_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_writes_unit')
        increase_throttled_by_provisioned_writes_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_provisioned_writes_scale')
        increase_throttled_by_consumed_writes_unit = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_writes_unit')
        increase_throttled_by_consumed_writes_scale = \
            get_table_option(
                key_name, 'increase_throttled_by_consumed_writes_scale')
        increase_consumed_writes_unit = \
            get_table_option(key_name, 'increase_consumed_writes_unit')
        increase_consumed_writes_with = \
            get_table_option(key_name, 'increase_consumed_writes_with')
        increase_consumed_writes_scale = \
            get_table_option(key_name, 'increase_consumed_writes_scale')
        decrease_consumed_writes_unit = \
            get_table_option(key_name, 'decrease_consumed_writes_unit')
        decrease_consumed_writes_with = \
            get_table_option(key_name, 'decrease_consumed_writes_with')
        decrease_consumed_writes_scale = \
            get_table_option(key_name, 'decrease_consumed_writes_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_write_units = current_write_units

    # Reset consecutive write count if num_write_checks_reset_percent
    # is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info('{0} - Resetting the number of consecutive '
                        'write checks. Reason: Consumed percent {1} is '
                        'greater than reset percent: {2}'.format(
                            table_name, consumed_write_units_percent,
                            num_write_checks_reset_percent))

            num_consec_write_checks = 0

    # Exit if up scaling has been disabled
    if not get_table_option(key_name, 'enable_writes_up_scaling'):
        logger.debug(
            '{0} - Up scaling event detected. No action taken as scaling '
            'up writes has been disabled in the configuration'.format(
                table_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_writes_unit = \
            increase_consumed_writes_unit or increase_writes_unit
        increase_throttled_by_provisioned_writes_unit = (
            increase_throttled_by_provisioned_writes_unit
            or increase_writes_unit)
        increase_throttled_by_consumed_writes_unit = \
            increase_throttled_by_consumed_writes_unit or increase_writes_unit

        increase_consumed_writes_with = \
            increase_consumed_writes_with or increase_writes_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_writes_scale,
            throttled_by_provisioned_write_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_writes_scale,
            throttled_by_consumed_write_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_writes_scale, consumed_write_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_writes_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_writes_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_writes_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        consumed_calculated_provisioning,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        elif (writes_upper_threshold
              and consumed_write_units_percent > writes_upper_threshold
              and not increase_consumed_writes_scale):

            if increase_consumed_writes_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_consumed_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        increase_consumed_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Increase needed due to high throttling
        if (throttled_writes_upper_threshold
                and throttled_write_count > throttled_writes_upper_threshold):

            if increase_writes_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        updated_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        updated_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = ("due to throttled events by provisioned "
                            "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning >
                calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = ("due to throttled events by consumed "
                            "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_write_units:
            logger.info('{0} - Resetting the number of consecutive '
                        'write checks. Reason: scale up {1}'.format(
                            table_name, scale_reason))
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if not update_needed:
        # If local/granular values not specified use global values
        decrease_consumed_writes_unit = \
            decrease_consumed_writes_unit or decrease_writes_unit

        decrease_consumed_writes_with = \
            decrease_consumed_writes_with or decrease_writes_with

        # Initialise variables to store calculated provisioning
        consumed_calculated_provisioning = scale_reader_decrease(
            decrease_consumed_writes_scale, consumed_write_units_percent)
        calculated_provisioning = None

        # Exit if down scaling has been disabled
        if not get_table_option(key_name, 'enable_writes_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down writes has been disabled in the configuration'.format(
                    table_name))
        # Exit if writes == 0% and downscaling has been disabled at 0%
        elif (consumed_write_units_percent == 0 and not get_table_option(
                key_name, 'allow_scaling_down_writes_on_0_percent')):
            logger.info(
                '{0} - Down scaling event detected. No action taken as scaling'
                ' down writes is not done when usage is at 0%'.format(
                    table_name))
        # Exit if writes are still throttled
        elif (throttled_writes_upper_threshold
              and throttled_write_count > throttled_writes_upper_threshold):
            logger.info(
                '{0} - Down scaling event detected. No action taken as there'
                ' are still throttled writes'.format(table_name))
        else:
            if consumed_calculated_provisioning:
                if decrease_consumed_writes_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_writes_in_percent(
                            updated_write_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_writes_in_units(
                            updated_write_units,
                            consumed_calculated_provisioning,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)
            elif (writes_lower_threshold
                  and consumed_write_units_percent < writes_lower_threshold
                  and not decrease_consumed_writes_scale):
                if decrease_consumed_writes_unit == 'percent':
                    calculated_provisioning = \
                        calculators.decrease_writes_in_percent(
                            updated_write_units,
                            decrease_consumed_writes_with,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)
                else:
                    calculated_provisioning = \
                        calculators.decrease_writes_in_units(
                            updated_write_units,
                            decrease_consumed_writes_with,
                            get_table_option(
                                key_name, 'min_provisioned_writes'),
                            table_name)

            if (calculated_provisioning
                    and current_write_units != calculated_provisioning):
                num_consec_write_checks += 1

                if num_consec_write_checks >= \
                        num_write_checks_before_scale_down:
                    update_needed = True
                    updated_write_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info('Will not increase writes over max-provisioned-writes '
                        'limit ({0} writes)'.format(updated_write_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_writes:
        if int(min_provisioned_writes) > int(updated_write_units):
            update_needed = True
            updated_write_units = int(min_provisioned_writes)
            logger.info(
                '{0} - Increasing writes to meet min-provisioned-writes '
                'limit ({1} writes)'.format(table_name, updated_write_units))

    if calculators.is_consumed_over_proposed(current_write_units,
                                             updated_write_units,
                                             consumed_write_units_percent):
        update_needed = False
        updated_write_units = current_write_units
        logger.info(
            '{0} - Consumed is over proposed write units. Will leave table at '
            'current setting.'.format(table_name))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name, num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
Esempio n. 6
0
def __ensure_provisioning_writes(table_name, key_name,
                                 num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_table_option(key_name, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - Autoscaling of writes has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_write_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(key_name,
                                                 'lookback_window_start')
        current_write_units = dynamodb.get_provisioned_table_write_units(
            table_name)
        consumed_write_units_percent = \
            table_stats.get_consumed_write_units_percent(
                table_name, lookback_window_start)
        throttled_write_count = \
            table_stats.get_throttled_write_event_count(
                table_name, lookback_window_start)
        writes_upper_threshold = \
            get_table_option(key_name, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_table_option(key_name, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_table_option(key_name, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_table_option(key_name, 'increase_writes_unit')
        increase_writes_with = \
            get_table_option(key_name, 'increase_writes_with')
        decrease_writes_unit = \
            get_table_option(key_name, 'decrease_writes_unit')
        decrease_writes_with = \
            get_table_option(key_name, 'decrease_writes_with')
        min_provisioned_writes = \
            get_table_option(key_name, 'min_provisioned_writes')
        max_provisioned_writes = \
            get_table_option(key_name, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_table_option(key_name, 'num_write_checks_before_scale_down')
        num_write_checks_reset_percent = \
            get_table_option(key_name, 'num_write_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_write_units = current_write_units

    # Reset consecutive write count num_write_checks_reset_percent is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info('{0} - Resetting the number of consecutive '
                        'write checks. Reason: Consumed percent {1} is '
                        'greater than reset percent: {2}'.format(
                            table_name, consumed_write_units_percent,
                            num_write_checks_reset_percent))

            num_consec_write_checks = 0

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_table_option(
            key_name, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - Scaling down writes is not done when usage is at 0%'.format(
                table_name))

    # Increase needed due to high CU consumption
    elif consumed_write_units_percent >= writes_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_writes_up_scaling'):
            logger.debug(
                '{0} - Up scaling event detected. No action taken as scaling '
                'up writes has been disabled in the configuration'.format(
                    table_name))
        else:
            if increase_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units, increase_writes_with,
                    get_table_option(key_name, 'max_provisioned_writes'),
                    consumed_write_units_percent, table_name)

            if current_write_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units, increase_writes_with,
                    get_table_option(key_name, 'max_provisioned_writes'),
                    consumed_write_units_percent, table_name)

            if current_write_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_write_units_percent <= writes_lower_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_writes_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling '
                'down writes has been disabled in the configuration'.format(
                    table_name))
        else:
            if decrease_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.decrease_writes_in_percent(
                        current_write_units,
                        decrease_writes_with,
                        get_table_option(key_name, 'min_provisioned_writes'),
                        table_name)
            else:
                calculated_provisioning = calculators.decrease_writes_in_units(
                    current_write_units, decrease_writes_with,
                    get_table_option(key_name, 'min_provisioned_writes'),
                    table_name)

            if current_write_units != calculated_provisioning:
                num_consec_write_checks = num_consec_write_checks + 1

                if (num_consec_write_checks >=
                        num_write_checks_before_scale_down):
                    update_needed = True
                    updated_write_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info('Will not increase writes over max-provisioned-writes '
                        'limit ({0} writes)'.format(updated_write_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_writes:
        if int(min_provisioned_writes) > int(updated_write_units):
            update_needed = True
            updated_write_units = int(min_provisioned_writes)
            logger.info(
                '{0} - Increasing writes to meet min-provisioned-writes '
                'limit ({1} writes)'.format(table_name, updated_write_units))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name, num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
Esempio n. 7
0
def __ensure_provisioning_writes(
        table_name, key_name, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_table_option(key_name, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - Autoscaling of writes has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_write_units(table_name), 0

    update_needed = False
    try:
        lookback_window_start = get_table_option(
            key_name, 'lookback_window_start')
        current_write_units = dynamodb.get_provisioned_table_write_units(
            table_name)
        consumed_write_units_percent = \
            table_stats.get_consumed_write_units_percent(
                table_name, lookback_window_start)
        throttled_write_count = \
            table_stats.get_throttled_write_event_count(
                table_name, lookback_window_start)
        writes_upper_threshold = \
            get_table_option(key_name, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_table_option(key_name, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_table_option(key_name, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_table_option(key_name, 'increase_writes_unit')
        increase_writes_with = \
            get_table_option(key_name, 'increase_writes_with')
        decrease_writes_unit = \
            get_table_option(key_name, 'decrease_writes_unit')
        decrease_writes_with = \
            get_table_option(key_name, 'decrease_writes_with')
        min_provisioned_writes = \
            get_table_option(key_name, 'min_provisioned_writes')
        max_provisioned_writes = \
            get_table_option(key_name, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_table_option(key_name, 'num_write_checks_before_scale_down')
        num_write_checks_reset_percent = \
            get_table_option(key_name, 'num_write_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_write_units = current_write_units

    # Reset consecutive write count num_write_checks_reset_percent is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info(
                '{0} - Resetting the number of consecutive '
                'write checks. Reason: Consumed percent {1} is '
                'greater than reset percent: {2}'.format(
                    table_name,
                    consumed_write_units_percent,
                    num_write_checks_reset_percent))

            num_consec_write_checks = 0

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not
            get_table_option(
                key_name, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - Scaling down writes is not done when usage is at 0%'.format(
                table_name))

    # Increase needed due to high CU consumption
    elif consumed_write_units_percent >= writes_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_writes_up_scaling'):
            logger.debug(
                '{0} - Up scaling event detected. No action taken as scaling '
                'up writes has been disabled in the configuration'.format(
                    table_name))
        else:
            if increase_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_table_option(key_name, 'max_provisioned_writes'),
                    consumed_write_units_percent,
                    table_name)

            if current_write_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_writes_with,
                        get_table_option(key_name, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        table_name)
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_table_option(key_name, 'max_provisioned_writes'),
                    consumed_write_units_percent,
                    table_name)

            if current_write_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_write_units_percent <= writes_lower_threshold:

        # Exit if up scaling has been disabled
        if not get_table_option(key_name, 'enable_writes_down_scaling'):
            logger.debug(
                '{0} - Down scaling event detected. No action taken as scaling '
                'down writes has been disabled in the configuration'.format(
                    table_name))
        else:
            if decrease_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.decrease_writes_in_percent(
                        current_write_units,
                        decrease_writes_with,
                        get_table_option(key_name, 'min_provisioned_writes'),
                        table_name)
            else:
                calculated_provisioning = calculators.decrease_writes_in_units(
                    current_write_units,
                    decrease_writes_with,
                    get_table_option(key_name, 'min_provisioned_writes'),
                    table_name)

            if current_write_units != calculated_provisioning:
                num_consec_write_checks = num_consec_write_checks + 1

                if (num_consec_write_checks >=
                        num_write_checks_before_scale_down):
                    update_needed = True
                    updated_write_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                'Will not increase writes over max-provisioned-writes '
                'limit ({0} writes)'.format(updated_write_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_writes:
        if int(min_provisioned_writes) > int(updated_write_units):
            update_needed = True
            updated_write_units = int(min_provisioned_writes)
            logger.info(
                '{0} - Increasing writes to meet min-provisioned-writes '
                'limit ({1} writes)'.format(table_name, updated_write_units))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name,
        num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
Esempio n. 8
0
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :returns: (bool, int) -- update_needed, updated_write_units
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of writes has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)

    update_needed = False
    try:
        updated_write_units = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
        consumed_write_units_percent = \
            gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
        throttled_write_count = \
            gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
        writes_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
        increase_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_with')
        decrease_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
        decrease_writes_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
        max_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down writes is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == 'percent':
            updated_provisioning = calculators.increase_writes_in_percent(
                updated_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            updated_provisioning = calculators.increase_writes_in_units(
                updated_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if updated_write_units != updated_provisioning:
            update_needed = True
            updated_write_units = updated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                updated_provisioning = calculators.increase_writes_in_percent(
                    updated_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                updated_provisioning = calculators.increase_writes_in_units(
                    updated_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if updated_write_units != updated_provisioning:
                update_needed = True
                updated_write_units = updated_provisioning

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == 'percent':
            updated_provisioning = calculators.decrease_writes_in_percent(
                updated_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            updated_provisioning = calculators.decrease_writes_in_units(
                updated_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if updated_write_units != updated_provisioning:
            update_needed = True
            updated_write_units = updated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - '
                'Will not increase writes over gsi-max-provisioned-writes '
                'limit ({2} writes)'.format(
                    table_name,
                    gsi_name,
                    updated_write_units))

    return update_needed, int(updated_write_units)
Esempio n. 9
0
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_gsi_option(table_key, gsi_key, "enable_writes_autoscaling"):
        logger.info("{0} - GSI: {1} - " "Autoscaling of writes has been disabled".format(table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name), 0

    update_needed = False
    try:
        current_write_units = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name)
        consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
        throttled_write_count = gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
        writes_upper_threshold = get_gsi_option(table_key, gsi_key, "writes_upper_threshold")
        writes_lower_threshold = get_gsi_option(table_key, gsi_key, "writes_lower_threshold")
        throttled_writes_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_writes_upper_threshold")
        increase_writes_unit = get_gsi_option(table_key, gsi_key, "increase_writes_unit")
        increase_writes_with = get_gsi_option(table_key, gsi_key, "increase_writes_with")
        decrease_writes_unit = get_gsi_option(table_key, gsi_key, "decrease_writes_unit")
        decrease_writes_with = get_gsi_option(table_key, gsi_key, "decrease_writes_with")
        max_provisioned_writes = get_gsi_option(table_key, gsi_key, "max_provisioned_writes")
        num_write_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_write_checks_before_scale_down")
        num_write_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_write_checks_reset_percent")
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current write unit value
    updated_write_units = current_write_units

    # Reset write consecutive count if num_write_checks_reset_percent is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info(
                "{0} - GSI: {1} - Resetting the number of consecutive "
                "write checks. Reason: Consumed percent {2} is "
                "greater than reset percent: {3}".format(
                    table_name, gsi_name, consumed_write_units_percent, num_write_checks_reset_percent
                )
            )

            num_consec_write_checks = 0

    # Check if we should update write provisioning
    if consumed_write_units_percent == 0 and not get_gsi_option(
        table_key, gsi_key, "allow_scaling_down_writes_on_0_percent"
    ):

        logger.info(
            "{0} - GSI: {1} - " "Scaling down writes is not done when usage is at 0%".format(table_name, gsi_name)
        )

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == "percent":
            calculated_provisioning = calculators.increase_writes_in_percent(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )
        else:
            calculated_provisioning = calculators.increase_writes_in_units(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )

        if current_write_units != calculated_provisioning:
            logger.info(
                "{0} - GSI: {1} - Resetting the number of consecutive "
                "write checks. Reason: scale up event detected".format(table_name, gsi_name)
            )
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calculated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == "percent":
                calculated_provisioning = calculators.increase_writes_in_percent(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                    "{0} - GSI: {1}".format(table_name, gsi_name),
                )
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                    "{0} - GSI: {1}".format(table_name, gsi_name),
                )

            if current_write_units != calculated_provisioning:
                logger.info(
                    "{0} - GSI: {1} - Resetting the number of consecutive "
                    "write checks. Reason: scale up event detected".format(table_name, gsi_name)
                )
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == "percent":
            calculated_provisioning = calculators.decrease_writes_in_percent(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, "min_provisioned_writes"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )
        else:
            calculated_provisioning = calculators.decrease_writes_in_units(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )

        if current_write_units != calculated_provisioning:
            num_consec_write_checks = num_consec_write_checks + 1

            if num_consec_write_checks >= num_write_checks_before_scale_down:
                update_needed = True
                updated_write_units = calculated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                "{0} - GSI: {1} - "
                "Will not increase writes over gsi-max-provisioned-writes "
                "limit ({2} writes)".format(table_name, gsi_name, updated_write_units)
            )

    logger.info(
        "{0} - GSI: {1} - Consecutive write checks {2}/{3}".format(
            table_name, gsi_name, num_consec_write_checks, num_write_checks_before_scale_down
        )
    )

    return update_needed, updated_write_units, num_consec_write_checks
Esempio n. 10
0
def __ensure_provisioning_writes(
        table_name, key_name, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type key_name: str
    :param key_name: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_table_option(key_name, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - Autoscaling of reads has been disabled'.format(table_name))
        return False, dynamodb.get_provisioned_table_write_units(table_name), 0

    update_needed = False
    try:
        current_write_units = dynamodb.get_provisioned_table_write_units(
            table_name)
        consumed_write_units_percent = \
            table_stats.get_consumed_write_units_percent(table_name)
        throttled_write_count = \
            table_stats.get_throttled_write_event_count(table_name)
        writes_upper_threshold = \
            get_table_option(key_name, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_table_option(key_name, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_table_option(key_name, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_table_option(key_name, 'increase_writes_unit')
        increase_writes_with = \
            get_table_option(key_name, 'increase_writes_with')
        decrease_writes_unit = \
            get_table_option(key_name, 'decrease_writes_unit')
        decrease_writes_with = \
            get_table_option(key_name, 'decrease_writes_with')
        max_provisioned_writes = \
            get_table_option(key_name, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_table_option(key_name, 'num_write_checks_before_scale_down')
	num_write_checks_reset_percent = \
	    get_table_option(key_name, 'num_write_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_write_units = current_write_units

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not
            get_table_option(
                key_name, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - Scaling down writes is not done when usage is at 0%'.format(
                table_name))

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == 'percent':
            calulated_provisioning = calculators.increase_writes_in_percent(
                current_write_units,
                increase_writes_with,
                get_table_option(key_name, 'max_provisioned_writes'),
                table_name)
        else:
            calulated_provisioning = calculators.increase_writes_in_units(
                current_write_units,
                increase_writes_with,
                get_table_option(key_name, 'max_provisioned_writes'),
                table_name)

        if current_write_units != calulated_provisioning:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'write checks. Reason: scale up event detected'.format(
                    table_name))
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calulated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                calulated_provisioning = calculators.increase_writes_in_percent(
                    current_write_units,
                    increase_writes_with,
                    get_table_option(key_name, 'max_provisioned_writes'),
                    table_name)
            else:
                calulated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_table_option(key_name, 'max_provisioned_writes'),
                    table_name)

            if current_write_units != calulated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calulated_provisioning
				
    elif consumed_write_units_percent >= num_write_checks_reset_percent:
		logger.info(
            '{0} - Resetting the number of consecutive '
            'write checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format(
                table_name, consumed_write_units_percent, num_write_checks_reset_percent))
		num_consec_write_checks = 0

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == 'percent':
            calulated_provisioning = calculators.decrease_writes_in_percent(
                current_write_units,
                decrease_writes_with,
                get_table_option(key_name, 'min_provisioned_writes'),
                table_name)
        else:
            calulated_provisioning = calculators.decrease_writes_in_units(
                current_write_units,
                decrease_writes_with,
                get_table_option(key_name, 'min_provisioned_writes'),
                table_name)

        if current_write_units != calulated_provisioning:
            num_consec_write_checks = num_consec_write_checks + 1

            if num_consec_write_checks >= num_write_checks_before_scale_down:
                update_needed = True
                updated_write_units = calulated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                'Will not increase writes over max-provisioned-writes '
                'limit ({0} writes)'.format(updated_write_units))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name,
        num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks