예제 #1
0
def publish_gsi_notification(
        table_key, gsi_key, message, message_types, subject=None):
    """ Publish a notification for a specific GSI

    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_key: str
    :param gsi_key: Table configuration option key name
    :type message: str
    :param message: Message to send via SNS
    :type message_types: list
    :param message_types: List with types: scale-up, scale-down, error-message
    :type subject: str
    :param subject: Subject to use for e-mail notifications
    :returns: None
    """
    topic = get_gsi_option(table_key, gsi_key, 'sns_topic_arn')
    if not topic:
        return

    for message_type in message_types:
        if (message_type in
                get_gsi_option(table_key, gsi_key, 'sns_message_types')):
            __publish(topic, message, subject)
            return
예제 #2
0
def publish_gsi_notification(
        table_key, gsi_key, message, message_types, subject=None):
    """ Publish a notification for a specific GSI

    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_key: str
    :param gsi_key: Table configuration option key name
    :type message: str
    :param message: Message to send via SNS
    :type message_types: list
    :param message_types:
        List with types:
        - scale-up
        - scale-down
        - high-throughput-alarm
        - low-throughput-alarm
    :type subject: str
    :param subject: Subject to use for e-mail notifications
    :returns: None
    """
    topic = get_gsi_option(table_key, gsi_key, 'sns_topic_arn')
    if not topic:
        return

    for message_type in message_types:
        if (message_type in
                get_gsi_option(table_key, gsi_key, 'sns_message_types')):
            __publish(topic, message, subject)
            return
예제 #3
0
def increase_reads_in_units(
        current_provisioning, units, table_name, table_key, gsi_name, gsi_key):
    """ Increase the current_provisioning with units units

    :type current_provisioning: int
    :param current_provisioning: The current provisioning
    :type units: int
    :param units: How many units should we increase with
    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Name of the key
    :returns: int -- New provisioning value
    """
    updated_provisioning = 0

    if int(units) > int(current_provisioning):
        updated_provisioning = 2 * int(current_provisioning)
    else:
        updated_provisioning = int(current_provisioning) + int(units)

    if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'):
        max_provisioned_reads = int(get_gsi_option(
            table_key, gsi_key, 'max_provisioned_reads'))
    else:
        max_provisioned_reads = 0

    if (max_provisioned_reads > 0 and
            updated_provisioning > max_provisioned_reads):
            logger.info(
                '{0} - GSI: {1} - '
                'Reached provisioned reads max limit: {2:d}'.format(
                    table_name,
                    gsi_name,
                    max_provisioned_reads))

            return max_provisioned_reads

    logger.debug(
        '{0} - GSI: {1} - '
        'Read provisioning will be increased to {2:d} units'.format(
            table_name,
            gsi_name,
            updated_provisioning))

    return updated_provisioning
예제 #4
0
def increase_writes_in_percent(
        current_provisioning, percent, table_name, table_key,
        gsi_name, gsi_key):
    """ Increase the current_provisioning with percent %

    :type current_provisioning: int
    :param current_provisioning: The current provisioning
    :type percent: int
    :param percent: How many percent should we increase with
    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Name of the key
    :returns: int -- New provisioning value
    """
    increase = int(math.ceil(float(current_provisioning)*(float(percent)/100)))
    updated_provisioning = current_provisioning + increase

    if get_gsi_option(table_key, gsi_key, 'max_provisioned_writes'):
        max_provisioned_writes = int(get_gsi_option(
            table_key, gsi_key, 'max_provisioned_writes'))
    else:
        max_provisioned_writes = 0

    if (max_provisioned_writes > 0 and
            updated_provisioning > max_provisioned_writes):

            logger.info(
                '{0} - GSI: {1} - '
                'Reached provisioned writes max limit: {2:d}'.format(
                    table_name,
                    gsi_name,
                    max_provisioned_writes))

            return max_provisioned_writes

    logger.debug(
        '{0} - GSI: {1}'
        'Write provisioning will be increased to {2:d} units'.format(
            table_name,
            gsi_name,
            updated_provisioning))

    return updated_provisioning
예제 #5
0
def increase_reads_in_percent(
        current_provisioning, percent, table_name, table_key,
        gsi_name, gsi_key):
    """ Increase the current_provisioning with percent %

    :type current_provisioning: int
    :param current_provisioning: The current provisioning
    :type percent: int
    :param percent: How many percent should we increase with
    :returns: int -- New provisioning value
    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Name of the key
    """
    increase = int(float(current_provisioning)*(float(percent)/100))
    updated_provisioning = current_provisioning + increase

    if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads') > 0:
        if (updated_provisioning > get_gsi_option(
                table_key, gsi_key, 'max_provisioned_reads')):

            logger.info(
                '{0} - GSI: {1} - '
                'Reached provisioned reads max limit: {2:d}'.format(
                    table_name,
                    gsi_name,
                    int(get_gsi_option(
                        table_key, gsi_key, 'max_provisioned_reads'))))

            return get_gsi_option(
                table_key, gsi_key, 'max_provisioned_reads')

    logger.debug(
        '{0} - GSI: {1} - '
        'Read provisioning will be increased to {2:d} units'.format(
            table_name,
            gsi_name,
            updated_provisioning))

    return updated_provisioning
예제 #6
0
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units,
                        write_units):
    """ Update throughput on the GSI

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type read_units: int
    :param read_units: New read unit provisioning
    :type write_units: int
    :param write_units: New write unit provisioning
    """
    try:
        current_ru = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        current_wu = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
    except JSONResponseError:
        raise

    # Check table status
    try:
        gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
    except JSONResponseError:
        raise

    logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
        table_name, gsi_name, gsi_status))
    if gsi_status != 'ACTIVE':
        logger.warning(
            '{0} - GSI: {1} - Not performing throughput changes when GSI '
            'status is {2}'.format(table_name, gsi_name, gsi_status))
        return

    # If this setting is True, we will only scale down when
    # BOTH reads AND writes are low
    if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
        read_units, write_units = __calculate_always_decrease_rw_values(
            table_name, gsi_name, read_units, current_ru, write_units,
            current_wu)

        if read_units == current_ru and write_units == current_wu:
            logger.info('{0} - GSI: {1} - No changes to perform'.format(
                table_name, gsi_name))
            return

    dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key,
                                     int(read_units), int(write_units))
예제 #7
0
def get_min_provisioned_reads(
        current_provisioning, table_name, table_key, gsi_name, gsi_key):
    """ Returns the minimum provisioned reads

    If the min_provisioned_reads value is less than current_provisioning * 2,
    then we return current_provisioning * 2, as DynamoDB cannot be scaled up
    with more than 100%.

    :type current_provisioning: int
    :param current_provisioning: The current provisioning
    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Name of the key
    :returns: int -- Minimum provisioned reads
    """
    min_provisioned_reads = 1

    if get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'):
        min_provisioned_reads = int(get_gsi_option(
            table_key, gsi_key, 'min_provisioned_reads'))

        if min_provisioned_reads > int(current_provisioning * 2):
            min_provisioned_reads = int(current_provisioning * 2)
            logger.debug(
                '{0} - GSI: {1} - '
                'Cannot reach min_provisioned_reads as max scale up '
                'is 100% of current provisioning'.format(
                    table_name, gsi_name))

    logger.debug(
        '{0} - GSI: {1} - '
        'Setting min provisioned reads to {2}'.format(
            table_name, gsi_name, min_provisioned_reads))

    return min_provisioned_reads
예제 #8
0
def publish_gsi_notification(table_key, gsi_key, message, message_types, subject=None):
    """ Publish a notification for a specific GSI

    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_key: str
    :param gsi_key: Table configuration option key name
    :type message: str
    :param message: Message to send via SNS
    :type message_types: list
    :param message_types: List with types: scale-up, scale-down, error-message
    :type subject: str
    :param subject: Subject to use for e-mail notifications
    :returns: None
    """
    topic = get_gsi_option(table_key, gsi_key, "sns_topic_arn")
    if not topic:
        return

    for message_type in message_types:
        if message_type in get_gsi_option(table_key, gsi_key, "sns_message_types"):
            __publish(topic, message, subject)
            return
예제 #9
0
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units):
    """ Update throughput on the GSI

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type read_units: int
    :param read_units: New read unit provisioning
    :type write_units: int
    :param write_units: New write unit provisioning
    """
    try:
        current_ru = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name)
        current_wu = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name)
    except JSONResponseError:
        raise

    # Check table status
    try:
        gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
    except JSONResponseError:
        raise

    logger.debug("{0} - GSI: {1} - GSI status is {2}".format(table_name, gsi_name, gsi_status))
    if gsi_status != "ACTIVE":
        logger.warning(
            "{0} - GSI: {1} - Not performing throughput changes when GSI "
            "status is {2}".format(table_name, gsi_name, gsi_status)
        )
        return

    # If this setting is True, we will only scale down when
    # BOTH reads AND writes are low
    if get_gsi_option(table_key, gsi_key, "always_decrease_rw_together"):
        read_units, write_units = __calculate_always_decrease_rw_values(
            table_name, gsi_name, read_units, current_ru, write_units, current_wu
        )

        if read_units == current_ru and write_units == current_wu:
            logger.info("{0} - GSI: {1} - No changes to perform".format(table_name, gsi_name))
            return

    dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
예제 #10
0
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :returns: (bool, int) -- update_needed, updated_read_units
    """
    update_needed = False
    updated_read_units = gsi_stats.get_provisioned_read_units(
        table_name, gsi_name)

    consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
        table_name, gsi_name)

    if (consumed_read_units_percent == 0 and not
            get_gsi_option(
                table_key,
                gsi_key,
                'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down reads is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif (consumed_read_units_percent >=
            get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')):

        if (get_gsi_option(table_key, gsi_key, 'increase_reads_unit') ==
                'percent'):
            updated_provisioning = calculators.increase_reads_in_percent(
                updated_read_units,
                get_gsi_option(table_key, gsi_key, 'increase_reads_with'),
                table_name,
                table_key,
                gsi_name,
                gsi_key,)
        else:
            updated_provisioning = calculators.increase_reads_in_units(
                updated_read_units,
                get_gsi_option(table_key, gsi_key, 'increase_reads_with'),
                table_name,
                table_key,
                gsi_name,
                gsi_key)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    elif (consumed_read_units_percent <=
            get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')):

        if (get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') ==
                'percent'):
            updated_provisioning = calculators.decrease_reads_in_percent(
                updated_read_units,
                get_gsi_option(table_key, gsi_key, 'decrease_reads_with'),
                table_name,
                table_key,
                gsi_name,
                gsi_key)
        else:
            updated_provisioning = calculators.decrease_reads_in_units(
                updated_read_units,
                get_gsi_option(table_key, gsi_key, 'decrease_reads_with'),
                table_name,
                table_key,
                gsi_name,
                gsi_key)

        if updated_read_units != updated_provisioning:
            update_needed = True
            updated_read_units = updated_provisioning

    if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'):
        if (int(updated_read_units) > int(
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'))):
            update_needed = True
            updated_read_units = int(
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'))
            logger.info(
                'Will not increase writes over gsi-max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    return update_needed, int(updated_read_units)
예제 #11
0
def __update_throughput(
        table_name, table_key, gsi_name, gsi_key, read_units, write_units):
    """ Update throughput on the GSI

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type read_units: int
    :param read_units: New read unit provisioning
    :type write_units: int
    :param write_units: New write unit provisioning
    """
    current_ru = gsi_stats.get_provisioned_read_units(
        table_name, gsi_name)
    current_wu = gsi_stats.get_provisioned_write_units(
        table_name, gsi_name)

    # Check that we are in the right time frame
    if get_gsi_option(table_key, gsi_key, 'maintenance_windows'):
        if (not __is_maintenance_window(table_name, gsi_name, get_gsi_option(
                table_key, gsi_key, 'maintenance_windows'))):

            logger.warning(
                '{0} - GSI: {1} - '
                'Current time is outside maintenance window'.format(
                    table_name,
                    gsi_name))
            return
        else:
            logger.info(
                '{0} - GSI: {1} - '
                'Current time is within maintenance window'.format(
                    table_name,
                    gsi_name))

    # Check table status
    gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
    logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
        table_name, gsi_name, gsi_status))
    if gsi_status != 'ACTIVE':
        logger.warning(
            '{0} - GSI: {1} - Not performing throughput changes when GSI '
            'status is {2}'.format(table_name, gsi_name, gsi_status))
        return

    # If this setting is True, we will only scale down when
    # BOTH reads AND writes are low
    if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
        if read_units < current_ru and write_units < current_wu:
            logger.debug(
                '{0} - GSI: {1} - '
                'Both reads and writes will be decreased'.format(
                    table_name,
                    gsi_name))
        elif read_units < current_ru:
            logger.info(
                '{0} - GSI: {1} - '
                'Will not decrease reads nor writes, waiting for '
                'both to become low before decrease'.format(
                    table_name, gsi_name))
            return
        elif write_units < current_wu:
            logger.info(
                '{0} - GSI: {1} - '
                'Will not decrease reads nor writes, waiting for '
                'both to become low before decrease'.format(
                    table_name, gsi_name))
            return

    if not get_global_option('dry_run'):
        dynamodb.update_gsi_provisioning(
            table_name,
            gsi_name,
            int(read_units),
            int(write_units))
        logger.info(
            '{0} - GSI: {1} - '
            'Provisioning updated to {2} reads and {3} writes'.format(
                table_name,
                gsi_name,
                read_units,
                write_units))
예제 #12
0
def __update_throughput(
        table_name, table_key, gsi_name, gsi_key, read_units, write_units):
    """ Update throughput on the GSI

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type read_units: int
    :param read_units: New read unit provisioning
    :type write_units: int
    :param write_units: New write unit provisioning
    """
    try:
        current_ru = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        current_wu = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
    except JSONResponseError:
        raise

    # Check that we are in the right time frame
    if get_gsi_option(table_key, gsi_key, 'maintenance_windows'):
        if (not __is_maintenance_window(table_name, gsi_name, get_gsi_option(
                table_key, gsi_key, 'maintenance_windows'))):

            logger.warning(
                '{0} - GSI: {1} - '
                'Current time is outside maintenance window'.format(
                    table_name,
                    gsi_name))
            return
        else:
            logger.info(
                '{0} - GSI: {1} - '
                'Current time is within maintenance window'.format(
                    table_name,
                    gsi_name))

    # Check table status
    try:
        gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
    except JSONResponseError:
        raise

    logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
        table_name, gsi_name, gsi_status))
    if gsi_status != 'ACTIVE':
        logger.warning(
            '{0} - GSI: {1} - Not performing throughput changes when GSI '
            'status is {2}'.format(table_name, gsi_name, gsi_status))
        return

    # If this setting is True, we will only scale down when
    # BOTH reads AND writes are low
    if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
        read_units, write_units = __calculate_always_decrease_rw_values(
            table_name,
            gsi_name,
            read_units,
            current_ru,
            write_units,
            current_wu)

        if read_units == current_ru and write_units == current_wu:
            logger.info('{0} - GSI: {1} - No changes to perform'.format(
                table_name, gsi_name))
            return

    if not get_global_option('dry_run'):
        dynamodb.update_gsi_provisioning(
            table_name,
            table_key,
            gsi_name,
            gsi_key,
            int(read_units),
            int(write_units))
        logger.info(
            '{0} - GSI: {1} - '
            'Provisioning updated to {2} reads and {3} writes'.format(
                table_name,
                gsi_name,
                read_units,
                write_units))
예제 #13
0
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :returns: (bool, int) -- update_needed, updated_write_units
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of writes has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)

    update_needed = False
    try:
        updated_write_units = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
        consumed_write_units_percent = \
            gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
        throttled_write_count = \
            gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
        writes_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
        increase_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_with')
        decrease_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
        decrease_writes_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
        max_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down writes is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == 'percent':
            updated_provisioning = calculators.increase_writes_in_percent(
                updated_write_units,
                increase_writes_with,
                table_name,
                table_key,
                gsi_name,
                gsi_key)
        else:
            updated_provisioning = calculators.increase_writes_in_units(
                updated_write_units,
                increase_writes_with,
                table_name,
                table_key,
                gsi_name,
                gsi_key)

        if updated_write_units != updated_provisioning:
            update_needed = True
            updated_write_units = updated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                updated_provisioning = calculators.increase_writes_in_percent(
                    updated_write_units,
                    increase_writes_with,
                    table_name,
                    table_key,
                    gsi_name,
                    gsi_key)
            else:
                updated_provisioning = calculators.increase_writes_in_units(
                    updated_write_units,
                    increase_writes_with,
                    table_name,
                    table_key,
                    gsi_name,
                    gsi_key)

            if updated_write_units != updated_provisioning:
                update_needed = True
                updated_write_units = updated_provisioning

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == 'percent':
            updated_provisioning = calculators.decrease_writes_in_percent(
                updated_write_units,
                decrease_writes_with,
                table_name,
                table_key,
                gsi_name,
                gsi_key)
        else:
            updated_provisioning = calculators.decrease_writes_in_units(
                updated_write_units,
                decrease_writes_with,
                table_name,
                table_key,
                gsi_name,
                gsi_key)

        if updated_write_units != updated_provisioning:
            update_needed = True
            updated_write_units = updated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - '
                'Will not increase writes over gsi-max-provisioned-writes '
                'limit ({2} writes)'.format(
                    table_name,
                    gsi_name,
                    updated_write_units))

    return update_needed, int(updated_write_units)
예제 #14
0
def __ensure_provisioning_writes(
        table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of writes has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        lookback_window_start = get_gsi_option(
            table_key, gsi_key, 'lookback_window_start')
        current_write_units = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
        consumed_write_units_percent = \
            gsi_stats.get_consumed_write_units_percent(
                table_name, gsi_name, lookback_window_start)
        throttled_write_count = \
            gsi_stats.get_throttled_write_event_count(
                table_name, gsi_name, lookback_window_start)
        throttled_by_provisioned_write_percent = \
            gsi_stats.get_throttled_by_provisioned_write_event_percent(
                table_name, gsi_name, lookback_window_start)
        throttled_by_consumed_write_percent = \
            gsi_stats.get_throttled_by_consumed_write_percent(
                table_name, gsi_name, lookback_window_start)
        writes_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
        increase_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_with')
        decrease_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
        decrease_writes_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
        min_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'min_provisioned_writes')
        max_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_write_checks_before_scale_down')
        num_write_checks_reset_percent = \
            get_gsi_option(table_key, gsi_key, 'num_write_checks_reset_percent')
        increase_throttled_by_provisioned_writes_unit = \
            get_gsi_option(
                table_key,
                gsi_key,
                'increase_throttled_by_provisioned_writes_unit')
        increase_throttled_by_provisioned_writes_scale = \
            get_gsi_option(
                table_key,
                gsi_key,
                'increase_throttled_by_provisioned_writes_scale')
        increase_throttled_by_consumed_writes_unit = \
            get_gsi_option(
                table_key,
                gsi_key,
                'increase_throttled_by_consumed_writes_unit')
        increase_throttled_by_consumed_writes_scale = \
            get_gsi_option(
                table_key,
                gsi_key,
                'increase_throttled_by_consumed_writes_scale')
        increase_consumed_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_consumed_writes_unit')
        increase_consumed_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_consumed_writes_with')
        increase_consumed_writes_scale = \
            get_gsi_option(table_key, gsi_key, 'increase_consumed_writes_scale')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current write unit value
    updated_write_units = current_write_units

    # Reset write consecutive count if num_write_checks_reset_percent is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info(
                '{0} - GSI: {1} - Resetting the number of consecutive '
                'write checks. Reason: Consumed percent {2} is '
                'greater than reset percent: {3}'.format(
                    table_name,
                    gsi_name,
                    consumed_write_units_percent,
                    num_write_checks_reset_percent))

            num_consec_write_checks = 0

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down writes is not done when usage is at 0%'.format(
                table_name, gsi_name))

    # Exit if up scaling has been disabled
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_up_scaling'):
        logger.debug(
            '{0} - GSI: {1} - Up scaling event detected. No action taken as '
            'scaling up writes has been disabled in the configuration'.format(
                table_name, gsi_name))

    else:

        # If local/granular values not specified use global values
        increase_consumed_writes_unit = \
            increase_consumed_writes_unit or increase_writes_unit
        increase_throttled_by_provisioned_writes_unit = (
            increase_throttled_by_provisioned_writes_unit
            or increase_writes_unit)
        increase_throttled_by_consumed_writes_unit = \
            increase_throttled_by_consumed_writes_unit or increase_writes_unit

        increase_consumed_writes_with = \
            increase_consumed_writes_with or increase_writes_with

        # Initialise variables to store calculated provisioning
        throttled_by_provisioned_calculated_provisioning = scale_reader(
            increase_throttled_by_provisioned_writes_scale,
            throttled_by_provisioned_write_percent)
        throttled_by_consumed_calculated_provisioning = scale_reader(
            increase_throttled_by_consumed_writes_scale,
            throttled_by_consumed_write_percent)
        consumed_calculated_provisioning = scale_reader(
            increase_consumed_writes_scale,
            consumed_write_units_percent)
        throttled_count_calculated_provisioning = 0
        calculated_provisioning = 0

        # Increase needed due to high throttled to provisioned ratio
        if throttled_by_provisioned_calculated_provisioning:

            if increase_throttled_by_provisioned_writes_unit == 'percent':
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                throttled_by_provisioned_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        throttled_by_provisioned_calculated_provisioning,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))

        # Increase needed due to high throttled to consumed ratio
        if throttled_by_consumed_calculated_provisioning:

            if increase_throttled_by_consumed_writes_unit == 'percent':
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                throttled_by_consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        throttled_by_consumed_calculated_provisioning,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))

        # Increase needed due to high CU consumption
        if consumed_calculated_provisioning:

            if increase_consumed_writes_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        consumed_calculated_provisioning,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        consumed_calculated_provisioning,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))

        elif (writes_upper_threshold
                and consumed_write_units_percent > writes_upper_threshold
                and not increase_consumed_writes_scale):

            if increase_consumed_writes_unit == 'percent':
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_consumed_writes_with,
                        get_gsi_option(
                            table_key,
                            gsi_key,
                            'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                consumed_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        current_write_units,
                        increase_consumed_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))

        # Increase needed due to high throttling
        if (throttled_writes_upper_threshold
                and throttled_write_count > throttled_writes_upper_threshold):

            if increase_writes_unit == 'percent':
                throttled_count_calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        updated_write_units,
                        increase_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                throttled_count_calculated_provisioning = \
                    calculators.increase_writes_in_units(
                        updated_write_units,
                        increase_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'max_provisioned_writes'),
                        consumed_write_units_percent,
                        '{0} - GSI: {1}'.format(table_name, gsi_name))

        # Determine which metric requires the most scaling
        if (throttled_by_provisioned_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_provisioned_calculated_provisioning
            scale_reason = (
                "due to throttled events by provisioned "
                "units threshold being exceeded")
        if (throttled_by_consumed_calculated_provisioning
                > calculated_provisioning):
            calculated_provisioning = \
                throttled_by_consumed_calculated_provisioning
            scale_reason = (
                "due to throttled events by consumed "
                "units threshold being exceeded")
        if consumed_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = consumed_calculated_provisioning
            scale_reason = "due to consumed threshold being exceeded"
        if throttled_count_calculated_provisioning > calculated_provisioning:
            calculated_provisioning = throttled_count_calculated_provisioning
            scale_reason = "due to throttled events threshold being exceeded"

        if calculated_provisioning > current_write_units:
            logger.info(
                '{0} - GSI: {1} - Resetting the number of consecutive '
                'write checks. Reason: scale up {2}'.format(
                    table_name, gsi_name, scale_reason))
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    if (consumed_write_units_percent
            <= writes_lower_threshold and not update_needed):

        # Exit if down scaling has been disabled
        if not get_gsi_option(table_key, gsi_key, 'enable_writes_down_scaling'):
            logger.debug(
                '{0} - GSI: {1} - Down scaling event detected. '
                'No action taken as scaling '
                'down writes has been disabled in the configuration'.format(
                    table_name, gsi_name))
        else:
            if decrease_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.decrease_writes_in_percent(
                        current_write_units,
                        decrease_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'min_provisioned_writes'),
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.decrease_writes_in_units(
                    current_write_units,
                    decrease_writes_with,
                    get_gsi_option(
                        table_key, gsi_key, 'min_provisioned_writes'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_write_units != calculated_provisioning:
                num_consec_write_checks = num_consec_write_checks + 1

                if (num_consec_write_checks >=
                        num_write_checks_before_scale_down):
                    update_needed = True
                    updated_write_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - '
                'Will not increase writes over gsi-max-provisioned-writes '
                'limit ({2} writes)'.format(
                    table_name,
                    gsi_name,
                    updated_write_units))

    # Ensure that we have met the min-provisioning
    if min_provisioned_writes:
        if int(min_provisioned_writes) > int(updated_write_units):
            update_needed = True
            updated_write_units = int(min_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - Increasing writes to'
                'meet gsi-min-provisioned-writes '
                'limit ({2} writes)'.format(
                    table_name,
                    gsi_name,
                    updated_write_units))

    logger.info('{0} - GSI: {1} - Consecutive write checks {2}/{3}'.format(
        table_name,
        gsi_name,
        num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
예제 #15
0
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning alarm threshold is not exceeded

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    """
    lookback_window_start = get_gsi_option(table_key, gsi_key,
                                           'lookback_window_start')
    consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
        table_name, gsi_name, lookback_window_start)
    consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(
        table_name, gsi_name, lookback_window_start)

    reads_upper_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold')
    reads_lower_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold')
    writes_upper_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold')
    writes_lower_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold')

    # Check upper alarm thresholds
    upper_alert_triggered = False
    upper_alert_message = []
    if (reads_upper_alarm_threshold > 0
            and consumed_read_units_percent >= reads_upper_alarm_threshold):
        upper_alert_triggered = True
        upper_alert_message.append(
            '{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
            'was greater than or equal to the upper alarm '
            'threshold {3:d}%\n'.format(table_name, gsi_name,
                                        consumed_read_units_percent,
                                        reads_upper_alarm_threshold))

    if (writes_upper_alarm_threshold > 0
            and consumed_write_units_percent >= writes_upper_alarm_threshold):
        upper_alert_triggered = True
        upper_alert_message.append(
            '{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
            'was greater than or equal to the upper alarm '
            'threshold {3:d}%\n'.format(table_name, gsi_name,
                                        consumed_write_units_percent,
                                        writes_upper_alarm_threshold))

    # Check lower alarm thresholds
    lower_alert_triggered = False
    lower_alert_message = []
    if (reads_lower_alarm_threshold > 0
            and consumed_read_units_percent < reads_lower_alarm_threshold):
        lower_alert_triggered = True
        lower_alert_message.append(
            '{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
            'was below the lower alarm threshold {3:d}%\n'.format(
                table_name, gsi_name, consumed_read_units_percent,
                reads_lower_alarm_threshold))

    if (writes_lower_alarm_threshold > 0
            and consumed_write_units_percent < writes_lower_alarm_threshold):
        lower_alert_triggered = True
        lower_alert_message.append(
            '{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
            'was below the lower alarm threshold {3:d}%\n'.format(
                table_name, gsi_name, consumed_write_units_percent,
                writes_lower_alarm_threshold))

    # Send alert if needed
    if upper_alert_triggered:
        logger.info(
            '{0} - GSI: {1} - Will send high provisioning alert'.format(
                table_name, gsi_name))
        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            ''.join(upper_alert_message), ['high-throughput-alarm'],
            subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format(
                table_name, gsi_name))
    elif lower_alert_triggered:
        logger.info('{0} - GSI: {1} - Will send low provisioning alert'.format(
            table_name, gsi_name))
        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            ''.join(lower_alert_message), ['low-throughput-alarm'],
            subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format(
                table_name, gsi_name))
    else:
        logger.debug(
            '{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format(
                table_name, gsi_name))
예제 #16
0
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning alarm threshold is not exceeded

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    """
    lookback_window_start = get_gsi_option(
        table_key, gsi_key, 'lookback_window_start')
    consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
        table_name, gsi_name, lookback_window_start)
    consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(
        table_name, gsi_name, lookback_window_start)

    reads_upper_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold')
    reads_lower_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold')
    writes_upper_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold')
    writes_lower_alarm_threshold = \
        get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold')

    # Check upper alarm thresholds
    upper_alert_triggered = False
    upper_alert_message = []
    if (reads_upper_alarm_threshold > 0 and
            consumed_read_units_percent >= reads_upper_alarm_threshold):
        upper_alert_triggered = True
        upper_alert_message.append(
            '{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
            'was greater than or equal to the upper alarm '
            'threshold {3:d}%\n'.format(
                table_name,
                gsi_name,
                consumed_read_units_percent,
                reads_upper_alarm_threshold))

    if (writes_upper_alarm_threshold > 0 and
            consumed_write_units_percent >= writes_upper_alarm_threshold):
        upper_alert_triggered = True
        upper_alert_message.append(
            '{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
            'was greater than or equal to the upper alarm '
            'threshold {3:d}%\n'.format(
                table_name,
                gsi_name,
                consumed_write_units_percent,
                writes_upper_alarm_threshold))

    # Check lower alarm thresholds
    lower_alert_triggered = False
    lower_alert_message = []
    if (reads_lower_alarm_threshold > 0 and
            consumed_read_units_percent < reads_lower_alarm_threshold):
        lower_alert_triggered = True
        lower_alert_message.append(
            '{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
            'was below the lower alarm threshold {3:d}%\n'.format(
                table_name,
                gsi_name,
                consumed_read_units_percent,
                reads_lower_alarm_threshold))

    if (writes_lower_alarm_threshold > 0 and
            consumed_write_units_percent < writes_lower_alarm_threshold):
        lower_alert_triggered = True
        lower_alert_message.append(
            '{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
            'was below the lower alarm threshold {3:d}%\n'.format(
                table_name,
                gsi_name,
                consumed_write_units_percent,
                writes_lower_alarm_threshold))

    # Send alert if needed
    if upper_alert_triggered:
        logger.info(
            '{0} - GSI: {1} - Will send high provisioning alert'.format(
                table_name, gsi_name))
        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            ''.join(upper_alert_message),
            ['high-throughput-alarm'],
            subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format(
                table_name, gsi_name))
    elif lower_alert_triggered:
        logger.info(
            '{0} - GSI: {1} - Will send low provisioning alert'.format(
                table_name, gsi_name))
        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            ''.join(lower_alert_message),
            ['low-throughput-alarm'],
            subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format(
                table_name, gsi_name))
    else:
        logger.debug(
            '{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format(
                table_name, gsi_name))
예제 #17
0
def is_open(table_name=None, table_key=None, gsi_name=None, gsi_key=None):
    """ Checks whether the circuit breaker is open

    :param table_name: Name of the table being checked
    :param table_key: Configuration key for table
    :param gsi_name: Name of the GSI being checked
    :param gsi_key: Configuration key for the GSI
    :returns: bool -- True if the circuit is open
    """
    logger.debug('Checking circuit breaker status')

    # Parse the URL to make sure it is OK
    pattern = re.compile(
        r'^(?P<scheme>http(s)?://)'
        r'((?P<username>.+):(?P<password>.+)@){0,1}'
        r'(?P<url>.*)$'
    )

    url = timeout = None
    if gsi_name:
        url = get_gsi_option(table_key, gsi_key, 'circuit_breaker_url')
        timeout = get_gsi_option(table_key, gsi_key, 'circuit_breaker_timeout')
    elif table_name:
        url = get_table_option(table_key, 'circuit_breaker_url')
        timeout = get_table_option(table_key, 'circuit_breaker_timeout')

    if not url:
        url = get_global_option('circuit_breaker_url')
        timeout = get_global_option('circuit_breaker_timeout')

    match = pattern.match(url)
    if not match:
        logger.error('Malformatted URL: {0}'.format(url))
        sys.exit(1)

    use_basic_auth = False
    if match.group('username') and match.group('password'):
        use_basic_auth = True

    # Make the actual URL to call
    auth = ()
    if use_basic_auth:
        url = '{scheme}{url}'.format(
            scheme=match.group('scheme'),
            url=match.group('url'))
        auth = (match.group('username'), match.group('password'))

    headers = {}
    if table_name:
        headers["x-table-name"] = table_name
    if gsi_name:
        headers["x-gsi-name"] = gsi_name

    # Make the actual request
    try:
        response = requests.get(
            url,
            auth=auth,
            timeout=timeout / 1000.00,
            headers=headers)
        if int(response.status_code) == 200:
            logger.info('Circuit breaker is closed')
            return False
        else:
            logger.warning(
                'Circuit breaker returned with status code {0:d}'.format(
                    response.status_code))

    except requests.exceptions.SSLError as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.Timeout as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.ConnectionError as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.HTTPError as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.TooManyRedirects as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except Exception as error:
        logger.error('Unhandled exception: {0}'.format(error))
        logger.error(
            'Please file a bug at '
            'https://github.com/sebdah/dynamic-dynamodb/issues')

    return True
예제 #18
0
def update_gsi_provisioning(
        table_name, table_key, gsi_name, gsi_key,
        reads, writes, retry_with_only_increase=False):
    """ Update provisioning on a global secondary index

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: GSI configuration option key name
    :type reads: int
    :param reads: Number of reads to provision
    :type writes: int
    :param writes: Number of writes to provision
    :type retry_with_only_increase: bool
    :param retry_with_only_increase: Set to True to ensure only increases
    """
    current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name))
    current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name))

    # Make sure we aren't scaling down if we turned off downscaling
    if (not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling') or
            not get_gsi_option(
                table_key, gsi_key, 'enable_writes_down_scaling')):
        if (not get_gsi_option(
                table_key, gsi_key, 'enable_reads_down_scaling') and
                current_reads > reads):
            reads = current_reads
        if (not get_gsi_option(
                table_key, gsi_key, 'enable_writes_down_scaling') and
                current_writes > writes):
            writes = current_writes

        # Return if we do not need to scale at all
        if reads == current_reads and writes == current_writes:
            logger.info(
                '{0} - No need to scale up reads nor writes'.format(
                    table_name))
            return

    if retry_with_only_increase:
        # Ensure that we are only doing increases
        if current_reads > reads:
            reads = current_reads
        if current_writes > writes:
            writes = current_writes

        # Return if we do not need to scale at all
        if reads == current_reads and writes == current_writes:
            logger.info(
                '{0} - GSI: {1} - No need to scale up reads nor writes'.format(
                    table_name, gsi_name))
            return

        logger.info(
            '{0} - GSI: {1} - Retrying to update provisioning, '
            'excluding any decreases. '
            'Setting new reads to {2} and new writes to {3}'.format(
                table_name, gsi_name, reads, writes))

    # Check that we are in the right time frame
    m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows')
    if m_windows:
        if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
            logger.warning(
                '{0} - GSI: {1} - We are outside a maintenace window. '
                'Will only perform up scaling activites'.format(
                    table_name,
                    gsi_name))

            # Ensure that we are only doing increases
            if current_reads > reads:
                reads = current_reads
            if current_writes > writes:
                writes = current_writes

            # Return if we do not need to scale up
            if reads == current_reads and writes == current_writes:
                logger.info(
                    '{0} - GSI: {1} - '
                    'No need to scale up reads nor writes'.format(
                        table_name,
                        gsi_name))
                return

        else:
            logger.info(
                '{0} - GSI: {1} - '
                'Current time is within maintenance window'.format(
                    table_name,
                    gsi_name))

    logger.info(
        '{0} - GSI: {1} - '
        'Updating provisioning to {2} reads and {3} writes'.format(
            table_name, gsi_name, reads, writes))

    # Return if dry-run
    if get_global_option('dry_run'):
        return

    try:
        DYNAMODB_CONNECTION.update_table(
            table_name=table_name,
            global_secondary_index_updates=[
                {
                    "Update": {
                        "IndexName": gsi_name,
                        "ProvisionedThroughput": {
                            "ReadCapacityUnits": reads,
                            "WriteCapacityUnits": writes
                        }
                    }
                }
            ])

        message = []
        if current_reads > reads:
            message.append(
                '{0} - GSI: {1} - Reads: DOWN from {2} to {3}\n'.format(
                    table_name, gsi_name, current_reads, reads))
        elif current_reads < reads:
            message.append(
                '{0} - GSI: {1} - Reads: UP from {2} to {3}\n'.format(
                    table_name, gsi_name, current_reads, reads))
        if current_writes > writes:
            message.append(
                '{0} - GSI: {1} - Writes: DOWN from {2} to {3}\n'.format(
                    table_name, gsi_name, current_writes, writes))
        elif current_writes < writes:
            message.append(
                '{0} - GSI: {1} - Writes: UP from {2} to {3}\n'.format(
                    table_name, gsi_name, current_writes, writes))

        # See if we should send notifications for scale-down, scale-up or both
        sns_message_types = []
        if current_reads > reads or current_writes > writes:
            sns_message_types.append('scale-down')
        if current_reads < reads or current_writes < writes:
            sns_message_types.append('scale-up')

        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            ''.join(message),
            sns_message_types,
            subject='Updated provisioning for GSI {0}'.format(gsi_name))

    except JSONResponseError as error:
        exception = error.body['__type'].split('#')[1]
        know_exceptions = ['LimitExceededException']
        if exception in know_exceptions:
            logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
                table_name, gsi_name, exception, error.body['message']))
        else:
            logger.error(
                (
                    '{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
                    'Please file a bug report at '
                    'https://github.com/sebdah/dynamic-dynamodb/issues'
                ).format(
                    table_name, gsi_name, exception, error.body['message']))

        if (not retry_with_only_increase and
                exception == 'LimitExceededException'):
            logger.info(
                '{0} - GSI: {1} - Will retry to update provisioning '
                'with only increases'.format(table_name, gsi_name))
            update_gsi_provisioning(
                table_name,
                table_key,
                gsi_name,
                gsi_key,
                reads,
                writes,
                retry_with_only_increase=True)
예제 #19
0
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_gsi_option(table_key, gsi_key, "enable_writes_autoscaling"):
        logger.info("{0} - GSI: {1} - " "Autoscaling of writes has been disabled".format(table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name), 0

    update_needed = False
    try:
        current_write_units = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name)
        consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
        throttled_write_count = gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
        writes_upper_threshold = get_gsi_option(table_key, gsi_key, "writes_upper_threshold")
        writes_lower_threshold = get_gsi_option(table_key, gsi_key, "writes_lower_threshold")
        throttled_writes_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_writes_upper_threshold")
        increase_writes_unit = get_gsi_option(table_key, gsi_key, "increase_writes_unit")
        increase_writes_with = get_gsi_option(table_key, gsi_key, "increase_writes_with")
        decrease_writes_unit = get_gsi_option(table_key, gsi_key, "decrease_writes_unit")
        decrease_writes_with = get_gsi_option(table_key, gsi_key, "decrease_writes_with")
        max_provisioned_writes = get_gsi_option(table_key, gsi_key, "max_provisioned_writes")
        num_write_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_write_checks_before_scale_down")
        num_write_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_write_checks_reset_percent")
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current write unit value
    updated_write_units = current_write_units

    # Reset write consecutive count if num_write_checks_reset_percent is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info(
                "{0} - GSI: {1} - Resetting the number of consecutive "
                "write checks. Reason: Consumed percent {2} is "
                "greater than reset percent: {3}".format(
                    table_name, gsi_name, consumed_write_units_percent, num_write_checks_reset_percent
                )
            )

            num_consec_write_checks = 0

    # Check if we should update write provisioning
    if consumed_write_units_percent == 0 and not get_gsi_option(
        table_key, gsi_key, "allow_scaling_down_writes_on_0_percent"
    ):

        logger.info(
            "{0} - GSI: {1} - " "Scaling down writes is not done when usage is at 0%".format(table_name, gsi_name)
        )

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == "percent":
            calculated_provisioning = calculators.increase_writes_in_percent(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )
        else:
            calculated_provisioning = calculators.increase_writes_in_units(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )

        if current_write_units != calculated_provisioning:
            logger.info(
                "{0} - GSI: {1} - Resetting the number of consecutive "
                "write checks. Reason: scale up event detected".format(table_name, gsi_name)
            )
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calculated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == "percent":
                calculated_provisioning = calculators.increase_writes_in_percent(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                    "{0} - GSI: {1}".format(table_name, gsi_name),
                )
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
                    "{0} - GSI: {1}".format(table_name, gsi_name),
                )

            if current_write_units != calculated_provisioning:
                logger.info(
                    "{0} - GSI: {1} - Resetting the number of consecutive "
                    "write checks. Reason: scale up event detected".format(table_name, gsi_name)
                )
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == "percent":
            calculated_provisioning = calculators.decrease_writes_in_percent(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, "min_provisioned_writes"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )
        else:
            calculated_provisioning = calculators.decrease_writes_in_units(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
                "{0} - GSI: {1}".format(table_name, gsi_name),
            )

        if current_write_units != calculated_provisioning:
            num_consec_write_checks = num_consec_write_checks + 1

            if num_consec_write_checks >= num_write_checks_before_scale_down:
                update_needed = True
                updated_write_units = calculated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                "{0} - GSI: {1} - "
                "Will not increase writes over gsi-max-provisioned-writes "
                "limit ({2} writes)".format(table_name, gsi_name, updated_write_units)
            )

    logger.info(
        "{0} - GSI: {1} - Consecutive write checks {2}/{3}".format(
            table_name, gsi_name, num_consec_write_checks, num_write_checks_before_scale_down
        )
    )

    return update_needed, updated_write_units, num_consec_write_checks
예제 #20
0
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False):
    """ Update provisioning on a global secondary index

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: GSI configuration option key name
    :type reads: int
    :param reads: Number of reads to provision
    :type writes: int
    :param writes: Number of writes to provision
    :type retry_with_only_increase: bool
    :param retry_with_only_increase: Set to True to ensure only increases
    """
    current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name))
    current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name))

    if retry_with_only_increase:
        # Ensure that we are only doing increases
        if current_reads > reads:
            reads = current_reads
        if current_writes > writes:
            writes = current_writes

        # Return if we do not need to scale at all
        if reads == current_reads and writes == current_writes:
            logger.info("{0} - GSI: {1} - No need to scale up reads nor writes".format(table_name, gsi_name))
            return

        logger.info(
            "{0} - GSI: {1} - Retrying to update provisioning, excluding any decreases. "
            "Setting new reads to {2} and new writes to {3}".format(table_name, gsi_name, reads, writes)
        )

    # Check that we are in the right time frame
    m_windows = get_gsi_option(table_key, gsi_key, "maintenance_windows")
    if m_windows:
        if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
            logger.warning(
                "{0} - GSI: {1} - We are outside a maintenace window. "
                "Will only perform up scaling activites".format(table_name, gsi_name)
            )

            # Ensure that we are only doing increases
            if current_reads > reads:
                reads = current_reads
            if current_writes > writes:
                writes = current_writes

            # Return if we do not need to scale up
            if reads == current_reads and writes == current_writes:
                logger.info("{0} - GSI: {1} - " "No need to scale up reads nor writes".format(table_name, gsi_name))
                return

        else:
            logger.info("{0} - GSI: {1} - " "Current time is within maintenance window".format(table_name, gsi_name))

    logger.info(
        "{0} - GSI: {1} - "
        "Updating provisioning to {2} reads and {3} writes".format(table_name, gsi_name, reads, writes)
    )

    # Return if dry-run
    if get_global_option("dry_run"):
        return

    try:
        DYNAMODB_CONNECTION.update_table(
            table_name=table_name,
            global_secondary_index_updates=[
                {
                    "Update": {
                        "IndexName": gsi_name,
                        "ProvisionedThroughput": {"ReadCapacityUnits": reads, "WriteCapacityUnits": writes},
                    }
                }
            ],
        )

        message = ("{0} - GSI: {1} - Provisioning updated to " "{2} reads and {3} writes").format(
            table_name, gsi_name, reads, writes
        )

        # See if we should send notifications for scale-down, scale-up or both
        sns_message_types = []
        if current_reads > reads or current_writes > current_writes:
            sns_message_types.append("scale-down")
        if current_reads < reads or current_writes < current_writes:
            sns_message_types.append("scale-up")

        sns.publish_gsi_notification(
            table_key, gsi_key, message, sns_message_types, subject="Updated provisioning for GSI {0}".format(gsi_name)
        )

    except JSONResponseError as error:
        exception = error.body["__type"].split("#")[1]
        know_exceptions = ["LimitExceededException"]
        if exception in know_exceptions:
            logger.warning("{0} - GSI: {1} - {2}: {3}".format(table_name, gsi_name, exception, error.body["message"]))
        else:
            logger.error(
                (
                    "{0} - GSI: {1} - Unhandled exception: {2}: {3}. "
                    "Please file a bug report at "
                    "https://github.com/sebdah/dynamic-dynamodb/issues"
                ).format(table_name, gsi_name, exception, error.body["message"])
            )

        if not retry_with_only_increase and exception == "LimitExceededException":
            logger.info(
                "{0} - GSI: {1} - Will retry to update provisioning " "with only increases".format(table_name, gsi_name)
            )
            update_gsi_provisioning(
                table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True
            )
예제 #21
0
def is_open(table_name=None, table_key=None, gsi_name=None, gsi_key=None):
    """ Checks whether the circuit breaker is open

    :param table_name: Name of the table being checked
    :param table_key: Configuration key for table
    :param gsi_name: Name of the GSI being checked
    :param gsi_key: Configuration key for the GSI
    :returns: bool -- True if the circuit is open
    """
    logger.debug('Checking circuit breaker status')

    # Parse the URL to make sure it is OK
    pattern = re.compile(r'^(?P<scheme>http(s)?://)'
                         r'((?P<username>.+):(?P<password>.+)@){0,1}'
                         r'(?P<url>.*)$')

    url = timeout = None
    if gsi_name:
        url = get_gsi_option(table_key, gsi_key, 'circuit_breaker_url')
        timeout = get_gsi_option(table_key, gsi_key, 'circuit_breaker_timeout')
    elif table_name:
        url = get_table_option(table_key, 'circuit_breaker_url')
        timeout = get_table_option(table_key, 'circuit_breaker_timeout')

    if not url:
        url = get_global_option('circuit_breaker_url')
        timeout = get_global_option('circuit_breaker_timeout')

    match = pattern.match(url)
    if not match:
        logger.error('Malformatted URL: {0}'.format(url))
        sys.exit(1)

    use_basic_auth = False
    if match.group('username') and match.group('password'):
        use_basic_auth = True

    # Make the actual URL to call
    auth = ()
    if use_basic_auth:
        url = '{scheme}{url}'.format(scheme=match.group('scheme'),
                                     url=match.group('url'))
        auth = (match.group('username'), match.group('password'))

    headers = {}
    if table_name:
        headers["x-table-name"] = table_name
    if gsi_name:
        headers["x-gsi-name"] = gsi_name

    # Make the actual request
    try:
        response = requests.get(url,
                                auth=auth,
                                timeout=timeout / 1000.00,
                                headers=headers)
        if int(response.status_code) == 200:
            logger.info('Circuit breaker is closed')
            return False
        else:
            logger.warning(
                'Circuit breaker returned with status code {0:d}'.format(
                    response.status_code))

    except requests.exceptions.SSLError as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.Timeout as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.ConnectionError as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.HTTPError as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except requests.exceptions.TooManyRedirects as error:
        logger.warning('Circuit breaker: {0}'.format(error))
    except Exception as error:
        logger.error('Unhandled exception: {0}'.format(error))
        logger.error('Please file a bug at '
                     'https://github.com/sebdah/dynamic-dynamodb/issues')

    return True
예제 #22
0
def update_gsi_provisioning(table_name,
                            table_key,
                            gsi_name,
                            gsi_key,
                            reads,
                            writes,
                            retry_with_only_increase=False):
    """ Update provisioning on a global secondary index

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: GSI configuration option key name
    :type reads: int
    :param reads: Number of reads to provision
    :type writes: int
    :param writes: Number of writes to provision
    :type retry_with_only_increase: bool
    :param retry_with_only_increase: Set to True to ensure only increases
    """
    current_reads = int(get_provisioned_table_read_units(table_name))
    current_writes = int(get_provisioned_table_write_units(table_name))

    if retry_with_only_increase:
        # Ensure that we are only doing increases
        if current_reads > reads:
            reads = current_reads
        if current_writes > writes:
            writes = current_writes

        logger.info(
            '{0} - Retrying to update provisioning, excluding any decreases. '
            'Setting new reads to {1} and new writes to {2}'.format(
                table_name, reads, writes))

    # Check that we are in the right time frame
    m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows')
    if m_windows:
        if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
            logger.warning(
                '{0} - GSI: {1} - We are outside a maintenace window. '
                'Will only perform up scaling activites'.format(
                    table_name, gsi_name))

            # Ensure that we are only doing increases
            if current_reads > reads:
                reads = current_reads
            if current_writes > writes:
                writes = current_writes

            # Return if we do not need to scale up
            if reads == current_reads and writes == current_writes:
                logger.info('{0} - GSI: {1} - '
                            'No need to scale up reads nor writes'.format(
                                table_name, gsi_name))
                return

        else:
            logger.info('{0} - GSI: {1} - '
                        'Current time is within maintenance window'.format(
                            table_name, gsi_name))

    logger.info('{0} - GSI: {1} - '
                'Updating provisioning to {2} reads and {3} writes'.format(
                    table_name, gsi_name, reads, writes))

    # Return if dry-run
    if get_global_option('dry_run'):
        return

    try:
        DYNAMODB_CONNECTION.update_table(table_name=table_name,
                                         global_secondary_index_updates=[{
                                             "Update": {
                                                 "IndexName": gsi_name,
                                                 "ProvisionedThroughput": {
                                                     "ReadCapacityUnits":
                                                     reads,
                                                     "WriteCapacityUnits":
                                                     writes
                                                 }
                                             }
                                         }])

        message = ('{0} - GSI: {1} - Provisioning updated to '
                   '{2} reads and {3} writes').format(table_name, gsi_name,
                                                      reads, writes)

        # See if we should send notifications for scale-down, scale-up or both
        sns_message_types = []
        if current_reads > reads or current_writes > current_writes:
            sns_message_types.append('scale-down')
        if current_reads < reads or current_writes < current_writes:
            sns_message_types.append('scale-up')

        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            message,
            sns_message_types,
            subject='Updated provisioning for GSI {0}'.format(gsi_name))

    except JSONResponseError as error:
        exception = error.body['__type'].split('#')[1]
        know_exceptions = ['LimitExceededException']
        if exception in know_exceptions:
            logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
                table_name, gsi_name, exception, error.body['message']))
        else:
            logger.error(
                ('{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
                 'Please file a bug report at '
                 'https://github.com/sebdah/dynamic-dynamodb/issues').format(
                     table_name, gsi_name, exception, error.body['message']))

        if (not retry_with_only_increase
                and exception == 'LimitExceededException'):
            logger.info('{0} - GSI: {1} - Will retry to update provisioning '
                        'with only increases'.format(table_name, gsi_name))
            update_gsi_provisioning(table_name,
                                    table_key,
                                    gsi_name,
                                    gsi_key,
                                    reads,
                                    writes,
                                    retry_with_only_increase=True)
예제 #23
0
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
    """ Ensure that provisioning alarm threshold is not exceeded

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    """
    consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
    consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)

    reads_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-upper-alarm-threshold")
    reads_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-lower-alarm-threshold")
    writes_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-upper-alarm-threshold")
    writes_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-lower-alarm-threshold")

    # Check upper alarm thresholds
    upper_alert_triggered = False
    upper_alert_message = []
    if reads_upper_alarm_threshold > 0 and consumed_read_units_percent >= reads_upper_alarm_threshold:
        upper_alert_triggered = True
        upper_alert_message.append(
            "{0} - GSI: {1} - Consumed Read Capacity {2:d}% "
            "was greater than or equal to the upper alarm "
            "threshold {3:d}%\n".format(table_name, gsi_name, consumed_read_units_percent, reads_upper_alarm_threshold)
        )

    if writes_upper_alarm_threshold > 0 and consumed_write_units_percent >= writes_upper_alarm_threshold:
        upper_alert_triggered = True
        upper_alert_message.append(
            "{0} - GSI: {1} - Consumed Write Capacity {2:d}% "
            "was greater than or equal to the upper alarm "
            "threshold {3:d}%\n".format(
                table_name, gsi_name, consumed_write_units_percent, writes_upper_alarm_threshold
            )
        )

    # Check lower alarm thresholds
    lower_alert_triggered = False
    lower_alert_message = []
    if reads_lower_alarm_threshold > 0 and consumed_read_units_percent < reads_lower_alarm_threshold:
        lower_alert_triggered = True
        lower_alert_message.append(
            "{0} - GSI: {1} - Consumed Read Capacity {2:d}% "
            "was below the lower alarm threshold {3:d}%\n".format(
                table_name, gsi_name, consumed_read_units_percent, reads_lower_alarm_threshold
            )
        )

    if writes_lower_alarm_threshold > 0 and consumed_write_units_percent < writes_lower_alarm_threshold:
        lower_alert_triggered = True
        lower_alert_message.append(
            "{0} - GSI: {1} - Consumed Write Capacity {2:d}% "
            "was below the lower alarm threshold {3:d}%\n".format(
                table_name, gsi_name, consumed_write_units_percent, writes_lower_alarm_threshold
            )
        )

    # Send alert if needed
    if upper_alert_triggered:
        logger.info("{0} - GSI: {1} - Will send high provisioning alert".format(table_name, gsi_name))
        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            "".join(upper_alert_message),
            ["high-throughput-alarm"],
            subject="ALARM: High Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name),
        )
    elif lower_alert_triggered:
        logger.info("{0} - GSI: {1} - Will send low provisioning alert".format(table_name, gsi_name))
        sns.publish_gsi_notification(
            table_key,
            gsi_key,
            "".join(lower_alert_message),
            ["low-throughput-alarm"],
            subject="ALARM: Low Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name),
        )
    else:
        logger.debug("{0} - GSI: {1} - Throughput alarm thresholds not crossed".format(table_name, gsi_name))
예제 #24
0
def __ensure_provisioning_reads(
        table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of reads has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        lookback_window_start = get_gsi_option(
            table_key, gsi_key, 'lookback_window_start')
        current_read_units = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        consumed_read_units_percent = \
            gsi_stats.get_consumed_read_units_percent(
                table_name, gsi_name, lookback_window_start)
        throttled_read_count = \
            gsi_stats.get_throttled_read_event_count(
                table_name, gsi_name, lookback_window_start)
        reads_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
        increase_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
        decrease_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
        increase_reads_with = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_with')
        decrease_reads_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
        throttled_reads_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_reads_upper_threshold')
        max_provisioned_reads = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    # Reset consecutive reads if num_read_checks_reset_percent is reached
    if num_read_checks_reset_percent:

        if consumed_read_units_percent >= num_read_checks_reset_percent:

            logger.info(
                '{0} - GSI: {1} - Resetting the number of consecutive '
                'read checks. Reason: Consumed percent {2} is '
                'greater than reset percent: {3}'.format(
                    table_name,
                    gsi_name,
                    consumed_read_units_percent,
                    num_read_checks_reset_percent))

            num_consec_read_checks = 0

    if (consumed_read_units_percent == 0 and not
            get_gsi_option(
                table_key,
                gsi_key,
                'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down reads is not done when usage is at 0%'.format(
                table_name, gsi_name))

    # Increase needed due to high CU consumption
    elif consumed_read_units_percent >= reads_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'):
            logger.debug(
                '{0} - GSI: {1} - Up scaling event detected. '
                'No action taken as scaling '
                'up reads has been disabled in the configuration'.format(
                    table_name, gsi_name))
        else:
            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:

            if increase_reads_unit == 'percent':
                calculated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.increase_reads_in_units(
                    current_read_units,
                    increase_reads_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_read_units != calculated_provisioning:
                logger.info(
                    '{0} - GSI: {1} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name, gsi_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_read_units_percent <= reads_lower_threshold:

        # Exit if down scaling has been disabled
        if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'):
            logger.debug(
                '{0} - GSI: {1} - Down scaling event detected. '
                'No action taken as scaling '
                'down reads has been disabled in the configuration'.format(
                    table_name, gsi_name))
        else:
            if decrease_reads_unit == 'percent':
                calculated_provisioning = calculators.decrease_reads_in_percent(
                    current_read_units,
                    decrease_reads_with,
                    get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.decrease_reads_in_units(
                    current_read_units,
                    decrease_reads_with,
                    get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_read_units != calculated_provisioning:
                # We need to look at how many times the num_consec_read_checks
                # integer has incremented and Compare to config file value
                num_consec_read_checks = num_consec_read_checks + 1

                if num_consec_read_checks >= num_read_checks_before_scale_down:
                    update_needed = True
                    updated_read_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over gsi-max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format(
        table_name,
        gsi_name,
        num_consec_read_checks,
        num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks
예제 #25
0
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key,
                                 num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
        logger.info('{0} - GSI: {1} - '
                    'Autoscaling of writes has been disabled'.format(
                        table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        lookback_window_start = get_gsi_option(table_key, gsi_key,
                                               'lookback_window_start')
        current_write_units = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
        consumed_write_units_percent = \
            gsi_stats.get_consumed_write_units_percent(
                table_name, gsi_name, lookback_window_start)
        throttled_write_count = \
            gsi_stats.get_throttled_write_event_count(
                table_name, gsi_name, lookback_window_start)
        writes_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
        increase_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_with')
        decrease_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
        decrease_writes_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
        max_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_write_checks_before_scale_down')
        num_write_checks_reset_percent = \
            get_gsi_option(table_key, gsi_key, 'num_write_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current write unit value
    updated_write_units = current_write_units

    # Reset write consecutive count if num_write_checks_reset_percent is reached
    if num_write_checks_reset_percent:

        if consumed_write_units_percent >= num_write_checks_reset_percent:

            logger.info('{0} - GSI: {1} - Resetting the number of consecutive '
                        'write checks. Reason: Consumed percent {2} is '
                        'greater than reset percent: {3}'.format(
                            table_name, gsi_name, consumed_write_units_percent,
                            num_write_checks_reset_percent))

            num_consec_write_checks = 0

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down writes is not done when usage is at 0%'.format(
                table_name, gsi_name))

    # Increase needed due to high CU consumption
    elif consumed_write_units_percent >= writes_upper_threshold:

        # Exit if up scaling has been disabled
        if not get_gsi_option(table_key, gsi_key, 'enable_writes_up_scaling'):
            logger.debug(
                '{0} - GSI: {1} - Up scaling event detected. '
                'No action taken as scaling '
                'up writes has been disabled in the configuration'.format(
                    table_name, gsi_name))
        else:
            if increase_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'max_provisioned_writes'),
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units, increase_writes_with,
                    get_gsi_option(table_key, gsi_key,
                                   'max_provisioned_writes'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_write_units != calculated_provisioning:
                logger.info(
                    '{0} - GSI: {1} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name, gsi_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    # Increase needed due to high throttling
    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.increase_writes_in_percent(
                        current_write_units,
                        increase_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'max_provisioned_writes'),
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.increase_writes_in_units(
                    current_write_units, increase_writes_with,
                    get_gsi_option(table_key, gsi_key,
                                   'max_provisioned_writes'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_write_units != calculated_provisioning:
                logger.info(
                    '{0} - GSI: {1} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name, gsi_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calculated_provisioning

    # Decrease needed due to low CU consumption
    elif consumed_write_units_percent <= writes_lower_threshold:

        # Exit if down scaling has been disabled
        if not get_gsi_option(table_key, gsi_key,
                              'enable_writes_down_scaling'):
            logger.debug(
                '{0} - GSI: {1} - Down scaling event detected. '
                'No action taken as scaling '
                'down writes has been disabled in the configuration'.format(
                    table_name, gsi_name))
        else:
            if decrease_writes_unit == 'percent':
                calculated_provisioning = \
                    calculators.decrease_writes_in_percent(
                        current_write_units,
                        decrease_writes_with,
                        get_gsi_option(
                            table_key, gsi_key, 'min_provisioned_writes'),
                        '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calculated_provisioning = calculators.decrease_writes_in_units(
                    current_write_units, decrease_writes_with,
                    get_gsi_option(table_key, gsi_key,
                                   'min_provisioned_writes'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_write_units != calculated_provisioning:
                num_consec_write_checks = num_consec_write_checks + 1

                if (num_consec_write_checks >=
                        num_write_checks_before_scale_down):
                    update_needed = True
                    updated_write_units = calculated_provisioning

    # Never go over the configured max provisioning
    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - '
                'Will not increase writes over gsi-max-provisioned-writes '
                'limit ({2} writes)'.format(table_name, gsi_name,
                                            updated_write_units))

    logger.info('{0} - GSI: {1} - Consecutive write checks {2}/{3}'.format(
        table_name, gsi_name, num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
예제 #26
0
def __ensure_provisioning_writes(
        table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
    """ Ensure that provisioning of writes is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_write_checks: int
    :param num_consec_write_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_write_units, num_consec_write_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
        logger.info(
            '{0} - GSI: {1} - '
            'Autoscaling of writes has been disabled'.format(
                table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        current_write_units = dynamodb.get_provisioned_gsi_write_units(
            table_name, gsi_name)
        consumed_write_units_percent = \
            gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
        throttled_write_count = \
            gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
        writes_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
        writes_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
        throttled_writes_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_writes_upper_threshold')
        increase_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
        increase_writes_with = \
            get_gsi_option(table_key, gsi_key, 'increase_writes_with')
        decrease_writes_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
        decrease_writes_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
        max_provisioned_writes = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
        num_write_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_write_checks_before_scale_down')
	num_write_checks_reset_percent = \
	    get_gsi_option(key_name, 'num_write_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current write unit value
    updated_write_units = current_write_units

    # Check if we should update write provisioning
    if (consumed_write_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down writes is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_write_units_percent >= writes_upper_threshold:

        if increase_writes_unit == 'percent':
            calulated_provisioning = calculators.increase_writes_in_percent(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.increase_writes_in_units(
                current_write_units,
                increase_writes_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_write_units != calulated_provisioning:
            logger.info(
                '{0} - Resetting the number of consecutive '
                'write checks. Reason: scale up event detected'.format(
                    table_name))
            num_consec_write_checks = 0
            update_needed = True
            updated_write_units = calulated_provisioning

    elif throttled_write_count > throttled_writes_upper_threshold:

        if throttled_writes_upper_threshold > 0:
            if increase_writes_unit == 'percent':
                calulated_provisioning = calculators.increase_writes_in_percent(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calulated_provisioning = calculators.increase_writes_in_units(
                    current_write_units,
                    increase_writes_with,
                    get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_write_units != calulated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'write checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_write_checks = 0
                update_needed = True
                updated_write_units = calulated_provisioning
				
    elif consumed_write_units_percent >= num_write_checks_reset_percent:
		logger.info(
            '{0} - Resetting the number of consecutive '
            'write checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format(
                table_name, consumed_write_units_percent, num_write_checks_reset_percent))
		num_consec_write_checks = 0

    elif consumed_write_units_percent <= writes_lower_threshold:

        if decrease_writes_unit == 'percent':
            calulated_provisioning = calculators.decrease_writes_in_percent(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.decrease_writes_in_units(
                current_write_units,
                decrease_writes_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_write_units != calulated_provisioning:
            num_consec_write_checks = num_consec_write_checks + 1

            if num_consec_write_checks >= num_write_checks_before_scale_down:
                update_needed = True
                updated_write_units = calulated_provisioning

    if max_provisioned_writes:
        if int(updated_write_units) > int(max_provisioned_writes):
            update_needed = True
            updated_write_units = int(max_provisioned_writes)
            logger.info(
                '{0} - GSI: {1} - '
                'Will not increase writes over gsi-max-provisioned-writes '
                'limit ({2} writes)'.format(
                    table_name,
                    gsi_name,
                    updated_write_units))

    logger.info('{0} - Consecutive write checks {1}/{2}'.format(
        table_name,
        num_consec_write_checks,
        num_write_checks_before_scale_down))

    return update_needed, updated_write_units, num_consec_write_checks
예제 #27
0
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key,
                                num_consec_read_checks):
    """ Ensure that provisioning is correct

    :type table_name: str
    :param table_name: Name of the DynamoDB table
    :type table_key: str
    :param table_key: Table configuration option key name
    :type gsi_name: str
    :param gsi_name: Name of the GSI
    :type gsi_key: str
    :param gsi_key: Configuration option key name
    :type num_consec_read_checks: int
    :param num_consec_read_checks: How many consecutive checks have we had
    :returns: (bool, int, int)
        update_needed, updated_read_units, num_consec_read_checks
    """
    if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
        logger.info('{0} - GSI: {1} - '
                    'Autoscaling of reads has been disabled'.format(
                        table_name, gsi_name))
        return False, dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name), 0

    update_needed = False
    try:
        current_read_units = dynamodb.get_provisioned_gsi_read_units(
            table_name, gsi_name)
        consumed_read_units_percent = \
            gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
        throttled_read_count = \
            gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
        reads_upper_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
        reads_lower_threshold = \
            get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
        increase_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
        decrease_reads_unit = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
        increase_reads_with = \
            get_gsi_option(table_key, gsi_key, 'increase_reads_with')
        decrease_reads_with = \
            get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
        throttled_reads_upper_threshold = \
            get_gsi_option(
                table_key, gsi_key, 'throttled_reads_upper_threshold')
        max_provisioned_reads = \
            get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
        num_read_checks_before_scale_down = \
            get_gsi_option(
                table_key, gsi_key, 'num_read_checks_before_scale_down')
        num_read_checks_reset_percent = \
            get_gsi_option(key_name, 'num_read_checks_reset_percent')
    except JSONResponseError:
        raise
    except BotoServerError:
        raise

    # Set the updated units to the current read unit value
    updated_read_units = current_read_units

    if (consumed_read_units_percent == 0 and not get_gsi_option(
            table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')):

        logger.info(
            '{0} - GSI: {1} - '
            'Scaling down reads is not done when usage is at 0%'.format(
                table_name, gsi_name))

    elif consumed_read_units_percent >= reads_upper_threshold:

        if increase_reads_unit == 'percent':
            calulated_provisioning = calculators.increase_reads_in_percent(
                current_read_units, increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.increase_reads_in_units(
                current_read_units, increase_reads_with,
                get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_read_units != calulated_provisioning:
            logger.info('{0} - Resetting the number of consecutive '
                        'read checks. Reason: scale up event detected'.format(
                            table_name))
            num_consec_read_checks = 0
            update_needed = True
            updated_read_units = calulated_provisioning

    elif throttled_read_count > throttled_reads_upper_threshold:

        if throttled_reads_upper_threshold > 0:

            if increase_reads_unit == 'percent':
                calulated_provisioning = calculators.increase_reads_in_percent(
                    current_read_units, increase_reads_with,
                    get_gsi_option(table_key, gsi_key,
                                   'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))
            else:
                calulated_provisioning = calculators.increase_reads_in_units(
                    current_read_units, increase_reads_with,
                    get_gsi_option(table_key, gsi_key,
                                   'max_provisioned_reads'),
                    '{0} - GSI: {1}'.format(table_name, gsi_name))

            if current_read_units != calulated_provisioning:
                logger.info(
                    '{0} - Resetting the number of consecutive '
                    'read checks. Reason: scale up event detected'.format(
                        table_name))
                num_consec_read_checks = 0
                update_needed = True
                updated_read_units = calulated_provisioning

    elif consumed_read_units_percent >= num_read_checks_reset_percent:
        logger.info(
            '{0} - Resetting the number of consecutive '
            'read checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'
            .format(table_name, consumed_read_units_percent,
                    num_read_checks_reset_percent))
        num_consec_read_checks = 0

    elif consumed_read_units_percent <= reads_lower_threshold:

        if decrease_reads_unit == 'percent':
            calulated_provisioning = calculators.decrease_reads_in_percent(
                current_read_units, decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))
        else:
            calulated_provisioning = calculators.decrease_reads_in_units(
                current_read_units, decrease_reads_with,
                get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
                '{0} - GSI: {1}'.format(table_name, gsi_name))

        if current_read_units != calulated_provisioning:
            # We need to look at how many times the num_consec_read_checks
            # integer has incremented and Compare to config file value
            num_consec_read_checks = num_consec_read_checks + 1

            if num_consec_read_checks >= num_read_checks_before_scale_down:
                update_needed = True
                updated_read_units = calulated_provisioning

    if max_provisioned_reads:
        if int(updated_read_units) > int(max_provisioned_reads):
            update_needed = True
            updated_read_units = int(max_provisioned_reads)
            logger.info(
                'Will not increase writes over gsi-max-provisioned-reads '
                'limit ({0} writes)'.format(updated_read_units))

    logger.info('{0} - Consecutive read checks {1}/{2}'.format(
        table_name, num_consec_read_checks, num_read_checks_before_scale_down))

    return update_needed, updated_read_units, num_consec_read_checks