def __ensure_provisioning_reads(
table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_read_units, num_consec_read_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of reads has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name), 0
update_needed = False
try:
current_read_units = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
consumed_read_units_percent = \
gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
throttled_read_count = \
gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
reads_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
reads_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
increase_reads_unit = \
get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
decrease_reads_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
increase_reads_with = \
get_gsi_option(table_key, gsi_key, 'increase_reads_with')
decrease_reads_with = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
throttled_reads_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_reads_upper_threshold')
max_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
num_read_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_read_checks_before_scale_down')
num_read_checks_reset_percent = \
get_gsi_option(key_name, 'num_read_checks_reset_percent')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current read unit value
updated_read_units = current_read_units
if (consumed_read_units_percent == 0 and not
get_gsi_option(
table_key,
gsi_key,
'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down reads is not done when usage is at 0%'.format(
table_name, gsi_name))
elif consumed_read_units_percent >= reads_upper_threshold:
if increase_reads_unit == 'percent':
calulated_provisioning = calculators.increase_reads_in_percent(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calulated_provisioning = calculators.increase_reads_in_units(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calulated_provisioning:
logger.info(
'{0} - Resetting the number of consecutive '
'read checks. Reason: scale up event detected'.format(
table_name))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calulated_provisioning
elif throttled_read_count > throttled_reads_upper_threshold:
if throttled_reads_upper_threshold > 0:
if increase_reads_unit == 'percent':
calulated_provisioning = calculators.increase_reads_in_percent(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calulated_provisioning = calculators.increase_reads_in_units(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calulated_provisioning:
logger.info(
'{0} - Resetting the number of consecutive '
'read checks. Reason: scale up event detected'.format(
table_name))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calulated_provisioning
elif consumed_read_units_percent >= num_read_checks_reset_percent:
logger.info(
'{0} - Resetting the number of consecutive '
'read checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format(
table_name, consumed_read_units_percent, num_read_checks_reset_percent))
num_consec_read_checks = 0
elif consumed_read_units_percent <= reads_lower_threshold:
if decrease_reads_unit == 'percent':
calulated_provisioning = calculators.decrease_reads_in_percent(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calulated_provisioning = calculators.decrease_reads_in_units(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calulated_provisioning:
# We need to look at how many times the num_consec_read_checks
# integer has incremented and Compare to config file value
num_consec_read_checks = num_consec_read_checks + 1
if num_consec_read_checks >= num_read_checks_before_scale_down:
update_needed = True
updated_read_units = calulated_provisioning
if max_provisioned_reads:
if int(updated_read_units) > int(max_provisioned_reads):
update_needed = True
updated_read_units = int(max_provisioned_reads)
logger.info(
'Will not increase writes over gsi-max-provisioned-reads '
'limit ({0} writes)'.format(updated_read_units))
logger.info('{0} - Consecutive read checks {1}/{2}'.format(
table_name,
num_consec_read_checks,
num_read_checks_before_scale_down))
return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:returns: (bool, int) -- update_needed, updated_read_units
"""
update_needed = False
updated_read_units = gsi_stats.get_provisioned_read_units(
table_name, gsi_name)
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name)
if (consumed_read_units_percent == 0 and not
get_gsi_option(
table_key,
gsi_key,
'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down reads is not done when usage is at 0%'.format(
table_name, gsi_name))
elif (consumed_read_units_percent >=
get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')):
if (get_gsi_option(table_key, gsi_key, 'increase_reads_unit') ==
'percent'):
updated_provisioning = calculators.increase_reads_in_percent(
updated_read_units,
get_gsi_option(table_key, gsi_key, 'increase_reads_with'),
table_name,
table_key,
gsi_name,
gsi_key,)
else:
updated_provisioning = calculators.increase_reads_in_units(
updated_read_units,
get_gsi_option(table_key, gsi_key, 'increase_reads_with'),
table_name,
table_key,
gsi_name,
gsi_key)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
elif (consumed_read_units_percent <=
get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')):
if (get_gsi_option(table_key, gsi_key, 'decrease_reads_unit') ==
'percent'):
updated_provisioning = calculators.decrease_reads_in_percent(
updated_read_units,
get_gsi_option(table_key, gsi_key, 'decrease_reads_with'),
table_name,
table_key,
gsi_name,
gsi_key)
else:
updated_provisioning = calculators.decrease_reads_in_units(
updated_read_units,
get_gsi_option(table_key, gsi_key, 'decrease_reads_with'),
table_name,
table_key,
gsi_name,
gsi_key)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'):
if (int(updated_read_units) > int(
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'))):
update_needed = True
updated_read_units = int(
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'))
logger.info(
'Will not increase writes over gsi-max-provisioned-reads '
'limit ({0} writes)'.format(updated_read_units))
return update_needed, int(updated_read_units)
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
"""
lookback_window_start = get_gsi_option(
table_key, gsi_key, 'lookback_window_start')
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
reads_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold')
reads_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold')
writes_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold')
writes_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold')
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if (reads_upper_alarm_threshold > 0 and
consumed_read_units_percent >= reads_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_read_units_percent,
reads_upper_alarm_threshold))
if (writes_upper_alarm_threshold > 0 and
consumed_write_units_percent >= writes_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_write_units_percent,
writes_upper_alarm_threshold))
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if (reads_lower_alarm_threshold > 0 and
consumed_read_units_percent < reads_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_read_units_percent,
reads_lower_alarm_threshold))
if (writes_lower_alarm_threshold > 0 and
consumed_write_units_percent < writes_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_write_units_percent,
writes_lower_alarm_threshold))
# Send alert if needed
if upper_alert_triggered:
logger.info(
'{0} - GSI: {1} - Will send high provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(upper_alert_message),
['high-throughput-alarm'],
subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
elif lower_alert_triggered:
logger.info(
'{0} - GSI: {1} - Will send low provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(lower_alert_message),
['low-throughput-alarm'],
subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
else:
logger.debug(
'{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format(
table_name, gsi_name))
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
"""
lookback_window_start = get_gsi_option(table_key, gsi_key,
'lookback_window_start')
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
reads_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold')
reads_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold')
writes_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold')
writes_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold')
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if (reads_upper_alarm_threshold > 0
and consumed_read_units_percent >= reads_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(table_name, gsi_name,
consumed_read_units_percent,
reads_upper_alarm_threshold))
if (writes_upper_alarm_threshold > 0
and consumed_write_units_percent >= writes_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(table_name, gsi_name,
consumed_write_units_percent,
writes_upper_alarm_threshold))
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if (reads_lower_alarm_threshold > 0
and consumed_read_units_percent < reads_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name, gsi_name, consumed_read_units_percent,
reads_lower_alarm_threshold))
if (writes_lower_alarm_threshold > 0
and consumed_write_units_percent < writes_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name, gsi_name, consumed_write_units_percent,
writes_lower_alarm_threshold))
# Send alert if needed
if upper_alert_triggered:
logger.info(
'{0} - GSI: {1} - Will send high provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(upper_alert_message), ['high-throughput-alarm'],
subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
elif lower_alert_triggered:
logger.info('{0} - GSI: {1} - Will send low provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(lower_alert_message), ['low-throughput-alarm'],
subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
else:
logger.debug(
'{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format(
table_name, gsi_name))
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:returns: (bool, int) -- update_needed, updated_read_units
"""
if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of reads has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
update_needed = False
try:
updated_read_units = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
consumed_read_units_percent = \
gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
throttled_read_count = \
gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
reads_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
reads_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
increase_reads_unit = \
get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
decrease_reads_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
increase_reads_with = \
get_gsi_option(table_key, gsi_key, 'increase_reads_with')
decrease_reads_with = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
throttled_reads_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_reads_upper_threshold')
max_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
except JSONResponseError:
raise
except BotoServerError:
raise
if (consumed_read_units_percent == 0 and not
get_gsi_option(
table_key,
gsi_key,
'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down reads is not done when usage is at 0%'.format(
table_name, gsi_name))
elif consumed_read_units_percent >= reads_upper_threshold:
if increase_reads_unit == 'percent':
updated_provisioning = calculators.increase_reads_in_percent(
updated_read_units,
increase_reads_with,
table_name,
table_key,
gsi_name,
gsi_key,)
else:
updated_provisioning = calculators.increase_reads_in_units(
updated_read_units,
increase_reads_with,
table_name,
table_key,
gsi_name,
gsi_key)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
elif throttled_read_count > throttled_reads_upper_threshold:
if throttled_reads_upper_threshold > 0:
if increase_reads_unit == 'percent':
updated_provisioning = calculators.increase_reads_in_percent(
updated_read_units,
increase_reads_with,
table_name,
table_key,
gsi_name,
gsi_key,)
else:
updated_provisioning = calculators.increase_reads_in_units(
updated_read_units,
increase_reads_with,
table_name,
table_key,
gsi_name,
gsi_key)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
elif consumed_read_units_percent <= reads_lower_threshold:
if decrease_reads_unit == 'percent':
updated_provisioning = calculators.decrease_reads_in_percent(
updated_read_units,
decrease_reads_with,
table_name,
table_key,
gsi_name,
gsi_key)
else:
updated_provisioning = calculators.decrease_reads_in_units(
updated_read_units,
decrease_reads_with,
table_name,
table_key,
gsi_name,
gsi_key)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
if max_provisioned_reads:
if int(updated_read_units) > int(max_provisioned_reads):
update_needed = True
updated_read_units = int(max_provisioned_reads)
logger.info(
'Will not increase writes over gsi-max-provisioned-reads '
'limit ({0} writes)'.format(updated_read_units))
return update_needed, int(updated_read_units)
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key,
num_consec_read_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_read_units, num_consec_read_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
logger.info('{0} - GSI: {1} - '
'Autoscaling of reads has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name), 0
update_needed = False
try:
lookback_window_start = get_gsi_option(table_key, gsi_key,
'lookback_window_start')
current_read_units = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
consumed_read_units_percent = \
gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
throttled_read_count = \
gsi_stats.get_throttled_read_event_count(
table_name, gsi_name, lookback_window_start)
reads_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
reads_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
increase_reads_unit = \
get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
decrease_reads_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
increase_reads_with = \
get_gsi_option(table_key, gsi_key, 'increase_reads_with')
decrease_reads_with = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
throttled_reads_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_reads_upper_threshold')
max_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
num_read_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_read_checks_before_scale_down')
num_read_checks_reset_percent = \
get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current read unit value
updated_read_units = current_read_units
# Reset consecutive reads if num_read_checks_reset_percent is reached
if num_read_checks_reset_percent:
if consumed_read_units_percent >= num_read_checks_reset_percent:
logger.info('{0} - GSI: {1} - Resetting the number of consecutive '
'read checks. Reason: Consumed percent {2} is '
'greater than reset percent: {3}'.format(
table_name, gsi_name, consumed_read_units_percent,
num_read_checks_reset_percent))
num_consec_read_checks = 0
if (consumed_read_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, 'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down reads is not done when usage is at 0%'.format(
table_name, gsi_name))
# Increase needed due to high CU consumption
elif consumed_read_units_percent >= reads_upper_threshold:
# Exit if up scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'):
logger.debug(
'{0} - GSI: {1} - Up scaling event detected. '
'No action taken as scaling '
'up reads has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if increase_reads_unit == 'percent':
calculated_provisioning = calculators.increase_reads_in_percent(
current_read_units, increase_reads_with,
get_gsi_option(table_key, gsi_key,
'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_reads_in_units(
current_read_units, increase_reads_with,
get_gsi_option(table_key, gsi_key,
'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
logger.info(
'{0} - Resetting the number of consecutive '
'read checks. Reason: scale up event detected'.format(
table_name))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
# Increase needed due to high throttling
elif throttled_read_count > throttled_reads_upper_threshold:
if throttled_reads_upper_threshold > 0:
if increase_reads_unit == 'percent':
calculated_provisioning = calculators.increase_reads_in_percent(
current_read_units, increase_reads_with,
get_gsi_option(table_key, gsi_key,
'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_reads_in_units(
current_read_units, increase_reads_with,
get_gsi_option(table_key, gsi_key,
'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'read checks. Reason: scale up event detected'.format(
table_name, gsi_name))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
# Decrease needed due to low CU consumption
elif consumed_read_units_percent <= reads_lower_threshold:
# Exit if down scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'):
logger.debug(
'{0} - GSI: {1} - Down scaling event detected. '
'No action taken as scaling '
'down reads has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if decrease_reads_unit == 'percent':
calculated_provisioning = calculators.decrease_reads_in_percent(
current_read_units, decrease_reads_with,
get_gsi_option(table_key, gsi_key,
'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.decrease_reads_in_units(
current_read_units, decrease_reads_with,
get_gsi_option(table_key, gsi_key,
'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
# We need to look at how many times the num_consec_read_checks
# integer has incremented and Compare to config file value
num_consec_read_checks = num_consec_read_checks + 1
if num_consec_read_checks >= num_read_checks_before_scale_down:
update_needed = True
updated_read_units = calculated_provisioning
# Never go over the configured max provisioning
if max_provisioned_reads:
if int(updated_read_units) > int(max_provisioned_reads):
update_needed = True
updated_read_units = int(max_provisioned_reads)
logger.info(
'Will not increase writes over gsi-max-provisioned-reads '
'limit ({0} writes)'.format(updated_read_units))
logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format(
table_name, gsi_name, num_consec_read_checks,
num_read_checks_before_scale_down))
return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
"""
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
reads_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-upper-alarm-threshold")
reads_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-lower-alarm-threshold")
writes_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-upper-alarm-threshold")
writes_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-lower-alarm-threshold")
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if reads_upper_alarm_threshold > 0 and consumed_read_units_percent >= reads_upper_alarm_threshold:
upper_alert_triggered = True
upper_alert_message.append(
"{0} - GSI: {1} - Consumed Read Capacity {2:d}% "
"was greater than or equal to the upper alarm "
"threshold {3:d}%\n".format(table_name, gsi_name, consumed_read_units_percent, reads_upper_alarm_threshold)
)
if writes_upper_alarm_threshold > 0 and consumed_write_units_percent >= writes_upper_alarm_threshold:
upper_alert_triggered = True
upper_alert_message.append(
"{0} - GSI: {1} - Consumed Write Capacity {2:d}% "
"was greater than or equal to the upper alarm "
"threshold {3:d}%\n".format(
table_name, gsi_name, consumed_write_units_percent, writes_upper_alarm_threshold
)
)
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if reads_lower_alarm_threshold > 0 and consumed_read_units_percent < reads_lower_alarm_threshold:
lower_alert_triggered = True
lower_alert_message.append(
"{0} - GSI: {1} - Consumed Read Capacity {2:d}% "
"was below the lower alarm threshold {3:d}%\n".format(
table_name, gsi_name, consumed_read_units_percent, reads_lower_alarm_threshold
)
)
if writes_lower_alarm_threshold > 0 and consumed_write_units_percent < writes_lower_alarm_threshold:
lower_alert_triggered = True
lower_alert_message.append(
"{0} - GSI: {1} - Consumed Write Capacity {2:d}% "
"was below the lower alarm threshold {3:d}%\n".format(
table_name, gsi_name, consumed_write_units_percent, writes_lower_alarm_threshold
)
)
# Send alert if needed
if upper_alert_triggered:
logger.info("{0} - GSI: {1} - Will send high provisioning alert".format(table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
"".join(upper_alert_message),
["high-throughput-alarm"],
subject="ALARM: High Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name),
)
elif lower_alert_triggered:
logger.info("{0} - GSI: {1} - Will send low provisioning alert".format(table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
"".join(lower_alert_message),
["low-throughput-alarm"],
subject="ALARM: Low Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name),
)
else:
logger.debug("{0} - GSI: {1} - Throughput alarm thresholds not crossed".format(table_name, gsi_name))
def __ensure_provisioning_reads(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_read_units, num_consec_read_checks
"""
if not get_gsi_option(table_key, gsi_key, "enable_reads_autoscaling"):
logger.info("{0} - GSI: {1} - " "Autoscaling of reads has been disabled".format(table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name), 0
update_needed = False
try:
current_read_units = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name)
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
throttled_read_count = gsi_stats.get_throttled_read_event_count(table_name, gsi_name)
reads_upper_threshold = get_gsi_option(table_key, gsi_key, "reads_upper_threshold")
reads_lower_threshold = get_gsi_option(table_key, gsi_key, "reads_lower_threshold")
increase_reads_unit = get_gsi_option(table_key, gsi_key, "increase_reads_unit")
decrease_reads_unit = get_gsi_option(table_key, gsi_key, "decrease_reads_unit")
increase_reads_with = get_gsi_option(table_key, gsi_key, "increase_reads_with")
decrease_reads_with = get_gsi_option(table_key, gsi_key, "decrease_reads_with")
throttled_reads_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_reads_upper_threshold")
max_provisioned_reads = get_gsi_option(table_key, gsi_key, "max_provisioned_reads")
num_read_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_read_checks_before_scale_down")
num_read_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_read_checks_reset_percent")
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current read unit value
updated_read_units = current_read_units
# Reset consecutive reads if num_read_checks_reset_percent is reached
if num_read_checks_reset_percent:
if consumed_read_units_percent >= num_read_checks_reset_percent:
logger.info(
"{0} - GSI: {1} - Resetting the number of consecutive "
"read checks. Reason: Consumed percent {2} is "
"greater than reset percent: {3}".format(
table_name, gsi_name, consumed_read_units_percent, num_read_checks_reset_percent
)
)
num_consec_read_checks = 0
if consumed_read_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, "allow_scaling_down_reads_on_0_percent"
):
logger.info(
"{0} - GSI: {1} - " "Scaling down reads is not done when usage is at 0%".format(table_name, gsi_name)
)
elif consumed_read_units_percent >= reads_upper_threshold:
if increase_reads_unit == "percent":
calculated_provisioning = calculators.increase_reads_in_percent(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
else:
calculated_provisioning = calculators.increase_reads_in_units(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
if current_read_units != calculated_provisioning:
logger.info(
"{0} - Resetting the number of consecutive "
"read checks. Reason: scale up event detected".format(table_name)
)
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
elif throttled_read_count > throttled_reads_upper_threshold:
if throttled_reads_upper_threshold > 0:
if increase_reads_unit == "percent":
calculated_provisioning = calculators.increase_reads_in_percent(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
else:
calculated_provisioning = calculators.increase_reads_in_units(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
if current_read_units != calculated_provisioning:
logger.info(
"{0} - GSI: {1} - Resetting the number of consecutive "
"read checks. Reason: scale up event detected".format(table_name, gsi_name)
)
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
elif consumed_read_units_percent <= reads_lower_threshold:
if decrease_reads_unit == "percent":
calculated_provisioning = calculators.decrease_reads_in_percent(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
else:
calculated_provisioning = calculators.decrease_reads_in_units(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
if current_read_units != calculated_provisioning:
# We need to look at how many times the num_consec_read_checks
# integer has incremented and Compare to config file value
num_consec_read_checks = num_consec_read_checks + 1
if num_consec_read_checks >= num_read_checks_before_scale_down:
update_needed = True
updated_read_units = calculated_provisioning
if max_provisioned_reads:
if int(updated_read_units) > int(max_provisioned_reads):
update_needed = True
updated_read_units = int(max_provisioned_reads)
logger.info(
"Will not increase writes over gsi-max-provisioned-reads "
"limit ({0} writes)".format(updated_read_units)
)
logger.info(
"{0} - GSI: {1} - Consecutive read checks {2}/{3}".format(
table_name, gsi_name, num_consec_read_checks, num_read_checks_before_scale_down
)
)
return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads(
table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_read_units, num_consec_read_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of reads has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name), 0
update_needed = False
try:
lookback_window_start = get_gsi_option(
table_key, gsi_key, 'lookback_window_start')
current_read_units = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
consumed_read_units_percent = \
gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
throttled_read_count = \
gsi_stats.get_throttled_read_event_count(
table_name, gsi_name, lookback_window_start)
reads_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
reads_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
increase_reads_unit = \
get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
decrease_reads_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
increase_reads_with = \
get_gsi_option(table_key, gsi_key, 'increase_reads_with')
decrease_reads_with = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
throttled_reads_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_reads_upper_threshold')
min_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads')
max_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
num_read_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_read_checks_before_scale_down')
num_read_checks_reset_percent = \
get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current read unit value
updated_read_units = current_read_units
# Reset consecutive reads if num_read_checks_reset_percent is reached
if num_read_checks_reset_percent:
if consumed_read_units_percent >= num_read_checks_reset_percent:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'read checks. Reason: Consumed percent {2} is '
'greater than reset percent: {3}'.format(
table_name,
gsi_name,
consumed_read_units_percent,
num_read_checks_reset_percent))
num_consec_read_checks = 0
if (consumed_read_units_percent == 0 and not
get_gsi_option(
table_key,
gsi_key,
'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down reads is not done when usage is at 0%'.format(
table_name, gsi_name))
# Increase needed due to high CU consumption
elif consumed_read_units_percent >= reads_upper_threshold:
# Exit if up scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'):
logger.debug(
'{0} - GSI: {1} - Up scaling event detected. '
'No action taken as scaling '
'up reads has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if increase_reads_unit == 'percent':
calculated_provisioning = calculators.increase_reads_in_percent(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_reads_in_units(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
logger.info(
'{0} - Resetting the number of consecutive '
'read checks. Reason: scale up event detected'.format(
table_name))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
# Increase needed due to high throttling
elif throttled_read_count > throttled_reads_upper_threshold:
if throttled_reads_upper_threshold > 0:
if increase_reads_unit == 'percent':
calculated_provisioning = calculators.increase_reads_in_percent(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_reads_in_units(
current_read_units,
increase_reads_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'read checks. Reason: scale up event detected'.format(
table_name, gsi_name))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
# Decrease needed due to low CU consumption
elif consumed_read_units_percent <= reads_lower_threshold:
# Exit if down scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'):
logger.debug(
'{0} - GSI: {1} - Down scaling event detected. '
'No action taken as scaling '
'down reads has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if decrease_reads_unit == 'percent':
calculated_provisioning = calculators.decrease_reads_in_percent(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.decrease_reads_in_units(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
# We need to look at how many times the num_consec_read_checks
# integer has incremented and Compare to config file value
num_consec_read_checks = num_consec_read_checks + 1
if num_consec_read_checks >= num_read_checks_before_scale_down:
update_needed = True
updated_read_units = calculated_provisioning
# Never go over the configured max provisioning
if max_provisioned_reads:
if int(updated_read_units) > int(max_provisioned_reads):
update_needed = True
updated_read_units = int(max_provisioned_reads)
logger.info(
'{0} - GSI: {1} - Will not increase writes over '
'gsi-max-provisioned-reads '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_read_units))
# Ensure that we have met the min-provisioning
if min_provisioned_reads:
if int(min_provisioned_reads) > int(updated_read_units):
update_needed = True
updated_read_units = int(min_provisioned_reads)
logger.info(
'{0} - GSI: {1} - Increasing reads to'
'meet gsi-min-provisioned-reads '
'limit ({2} reads)'.format(
table_name,
gsi_name,
updated_read_units))
logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format(
table_name,
gsi_name,
num_consec_read_checks,
num_read_checks_before_scale_down))
return update_needed, updated_read_units, num_consec_read_checks
def __ensure_provisioning_reads(
table_name, table_key, gsi_name, gsi_key, num_consec_read_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_read_units, num_consec_read_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_reads_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of reads has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name), 0
update_needed = False
try:
lookback_window_start = get_gsi_option(
table_key, gsi_key, 'lookback_window_start')
current_read_units = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
consumed_read_units_percent = \
gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
throttled_read_count = \
gsi_stats.get_throttled_read_event_count(
table_name, gsi_name, lookback_window_start)
throttled_by_provisioned_read_percent = \
gsi_stats.get_throttled_by_provisioned_read_event_percent(
table_name, gsi_name, lookback_window_start)
throttled_by_consumed_read_percent = \
gsi_stats.get_throttled_by_consumed_read_percent(
table_name, gsi_name, lookback_window_start)
reads_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_upper_threshold')
reads_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'reads_lower_threshold')
increase_reads_unit = \
get_gsi_option(table_key, gsi_key, 'increase_reads_unit')
decrease_reads_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_unit')
increase_reads_with = \
get_gsi_option(table_key, gsi_key, 'increase_reads_with')
decrease_reads_with = \
get_gsi_option(table_key, gsi_key, 'decrease_reads_with')
throttled_reads_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_reads_upper_threshold')
min_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads')
max_provisioned_reads = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads')
num_read_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_read_checks_before_scale_down')
num_read_checks_reset_percent = \
get_gsi_option(table_key, gsi_key, 'num_read_checks_reset_percent')
increase_throttled_by_provisioned_reads_unit = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_provisioned_reads_unit')
increase_throttled_by_provisioned_reads_scale = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_provisioned_reads_scale')
increase_throttled_by_consumed_reads_unit = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_consumed_reads_unit')
increase_throttled_by_consumed_reads_scale = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_consumed_reads_scale')
increase_consumed_reads_unit = \
get_gsi_option(table_key, gsi_key, 'increase_consumed_reads_unit')
increase_consumed_reads_with = \
get_gsi_option(table_key, gsi_key, 'increase_consumed_reads_with')
increase_consumed_reads_scale = \
get_gsi_option(table_key, gsi_key, 'increase_consumed_reads_scale')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current read unit value
updated_read_units = current_read_units
# Reset consecutive reads if num_read_checks_reset_percent is reached
if num_read_checks_reset_percent:
if consumed_read_units_percent >= num_read_checks_reset_percent:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'read checks. Reason: Consumed percent {2} is '
'greater than reset percent: {3}'.format(
table_name,
gsi_name,
consumed_read_units_percent,
num_read_checks_reset_percent))
num_consec_read_checks = 0
if (consumed_read_units_percent == 0 and not
get_gsi_option(
table_key,
gsi_key,
'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down reads is not done when usage is at 0%'.format(
table_name, gsi_name))
# Exit if up scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_reads_up_scaling'):
logger.debug(
'{0} - GSI: {1} - Up scaling event detected. No action taken as '
'scaling up reads has been disabled in the configuration'.format(
table_name, gsi_name))
else:
# If local/granular values not specified use global values
increase_consumed_reads_unit = \
increase_consumed_reads_unit or increase_reads_unit
increase_throttled_by_provisioned_reads_unit = \
increase_throttled_by_provisioned_reads_unit or increase_reads_unit
increase_throttled_by_consumed_reads_unit = \
increase_throttled_by_consumed_reads_unit or increase_reads_unit
increase_consumed_reads_with = \
increase_consumed_reads_with or increase_reads_with
# Initialise variables to store calculated provisioning
throttled_by_provisioned_calculated_provisioning = scale_reader(
increase_throttled_by_provisioned_reads_scale,
throttled_by_provisioned_read_percent)
throttled_by_consumed_calculated_provisioning = scale_reader(
increase_throttled_by_consumed_reads_scale,
throttled_by_consumed_read_percent)
consumed_calculated_provisioning = scale_reader(
increase_consumed_reads_scale,
consumed_read_units_percent)
throttled_count_calculated_provisioning = 0
calculated_provisioning = 0
# Increase needed due to high throttled to provisioned ratio
if throttled_by_provisioned_calculated_provisioning:
if increase_throttled_by_provisioned_reads_unit == 'percent':
throttled_by_provisioned_calculated_provisioning = \
calculators.increase_reads_in_percent(
current_read_units,
throttled_by_provisioned_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
throttled_by_provisioned_calculated_provisioning = \
calculators.increase_reads_in_units(
current_read_units,
throttled_by_provisioned_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Increase needed due to high throttled to consumed ratio
if throttled_by_consumed_calculated_provisioning:
if increase_throttled_by_consumed_reads_unit == 'percent':
throttled_by_consumed_calculated_provisioning = \
calculators.increase_reads_in_percent(
current_read_units,
throttled_by_consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
throttled_by_consumed_calculated_provisioning = \
calculators.increase_reads_in_units(
current_read_units,
throttled_by_consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Increase needed due to high CU consumption
if consumed_calculated_provisioning:
if increase_consumed_reads_unit == 'percent':
consumed_calculated_provisioning = \
calculators.increase_reads_in_percent(
current_read_units,
consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
consumed_calculated_provisioning = \
calculators.increase_reads_in_units(
current_read_units,
consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
elif (reads_upper_threshold
and consumed_read_units_percent > reads_upper_threshold
and not increase_consumed_reads_scale):
if increase_consumed_reads_unit == 'percent':
consumed_calculated_provisioning = \
calculators.increase_reads_in_percent(
current_read_units,
increase_consumed_reads_with,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
consumed_calculated_provisioning = \
calculators.increase_reads_in_units(
current_read_units,
increase_consumed_reads_with,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Increase needed due to high throttling
if (throttled_reads_upper_threshold
and throttled_read_count > throttled_reads_upper_threshold):
if increase_reads_unit == 'percent':
throttled_count_calculated_provisioning = \
calculators.increase_reads_in_percent(
updated_read_units,
increase_reads_with,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
throttled_count_calculated_provisioning = \
calculators.increase_reads_in_units(
updated_read_units,
increase_reads_with,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_reads'),
consumed_read_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Determine which metric requires the most scaling
if (throttled_by_provisioned_calculated_provisioning
> calculated_provisioning):
calculated_provisioning = \
throttled_by_provisioned_calculated_provisioning
scale_reason = (
"due to throttled events by provisioned "
"units threshold being exceeded")
if (throttled_by_consumed_calculated_provisioning
> calculated_provisioning):
calculated_provisioning = \
throttled_by_consumed_calculated_provisioning
scale_reason = (
"due to throttled events by consumed "
"units threshold being exceeded")
if consumed_calculated_provisioning > calculated_provisioning:
calculated_provisioning = consumed_calculated_provisioning
scale_reason = "due to consumed threshold being exceeded"
if throttled_count_calculated_provisioning > calculated_provisioning:
calculated_provisioning = throttled_count_calculated_provisioning
scale_reason = "due to throttled events threshold being exceeded"
if calculated_provisioning > current_read_units:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'read checks. Reason: scale up {2}'.format(
table_name, gsi_name, scale_reason))
num_consec_read_checks = 0
update_needed = True
updated_read_units = calculated_provisioning
# Decrease needed due to low CU consumption
if (consumed_read_units_percent <= reads_lower_threshold
and not update_needed):
# Exit if down scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling'):
logger.debug(
'{0} - GSI: {1} - Down scaling event detected. '
'No action taken as scaling '
'down reads has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if decrease_reads_unit == 'percent':
calculated_provisioning = calculators.decrease_reads_in_percent(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.decrease_reads_in_units(
current_read_units,
decrease_reads_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_read_units != calculated_provisioning:
# We need to look at how many times the num_consec_read_checks
# integer has incremented and Compare to config file value
num_consec_read_checks = num_consec_read_checks + 1
if num_consec_read_checks >= num_read_checks_before_scale_down:
update_needed = True
updated_read_units = calculated_provisioning
# Never go over the configured max provisioning
if max_provisioned_reads:
if int(updated_read_units) > int(max_provisioned_reads):
update_needed = True
updated_read_units = int(max_provisioned_reads)
logger.info(
'{0} - GSI: {1} - Will not increase writes over '
'gsi-max-provisioned-reads '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_read_units))
# Ensure that we have met the min-provisioning
if min_provisioned_reads:
if int(min_provisioned_reads) > int(updated_read_units):
update_needed = True
updated_read_units = int(min_provisioned_reads)
logger.info(
'{0} - GSI: {1} - Increasing reads to'
'meet gsi-min-provisioned-reads '
'limit ({2} reads)'.format(
table_name,
gsi_name,
updated_read_units))
logger.info('{0} - GSI: {1} - Consecutive read checks {2}/{3}'.format(
table_name,
gsi_name,
num_consec_read_checks,
num_read_checks_before_scale_down))
return update_needed, updated_read_units, num_consec_read_checks
