def get_consumed_write_units_percent(table_name, gsi_name, time_frame=300):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, time_frame,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) / float(
dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)) * 100))
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed write units: {2:d}%'.format(
table_name, gsi_name, consumed_write_units_percent))
return consumed_write_units_percent
def get_consumed_write_units_percent(table_name, gsi_name, time_frame=300):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(
table_name, gsi_name, time_frame, 'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)) * 100))
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed write units: {2:d}%'.format(
table_name, gsi_name, consumed_write_units_percent))
return consumed_write_units_percent
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units,
write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
current_wu = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
except JSONResponseError:
raise
# Check table status
try:
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
except JSONResponseError:
raise
logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
table_name, gsi_name, gsi_status))
if gsi_status != 'ACTIVE':
logger.warning(
'{0} - GSI: {1} - Not performing throughput changes when GSI '
'status is {2}'.format(table_name, gsi_name, gsi_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name, gsi_name, read_units, current_ru, write_units,
current_wu)
if read_units == current_ru and write_units == current_wu:
logger.info('{0} - GSI: {1} - No changes to perform'.format(
table_name, gsi_name))
return
dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key,
int(read_units), int(write_units))
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name)
current_wu = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name)
except JSONResponseError:
raise
# Check table status
try:
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
except JSONResponseError:
raise
logger.debug("{0} - GSI: {1} - GSI status is {2}".format(table_name, gsi_name, gsi_status))
if gsi_status != "ACTIVE":
logger.warning(
"{0} - GSI: {1} - Not performing throughput changes when GSI "
"status is {2}".format(table_name, gsi_name, gsi_status)
)
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, "always_decrease_rw_together"):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name, gsi_name, read_units, current_ru, write_units, current_wu
)
if read_units == current_ru and write_units == current_wu:
logger.info("{0} - GSI: {1} - No changes to perform".format(table_name, gsi_name))
return
dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def get_throttled_by_provisioned_write_event_percent(
table_name, gsi_name, lookback_window_start=15, lookback_period=5):
""" Returns the number of throttled write events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: float -- Percent of throttled write events by provisioning
"""
try:
metrics = __get_aws_metric(
table_name,
gsi_name,
lookback_window_start,
lookback_period,
'WriteThrottleEvents')
except BotoServerError:
raise
if metrics:
lookback_seconds = lookback_period * 60
throttled_write_events = float(metrics[0]['Sum']) / float(
lookback_seconds)
else:
throttled_write_events = 0
try:
gsi_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
throttled_by_provisioned_write_percent = (
float(throttled_write_events) /
float(gsi_write_units) * 100)
except JSONResponseError:
raise
logger.info(
'{0} - GSI: {1} - Throttled write percent '
'by provision: {2:.2f}%'.format(
table_name, gsi_name, throttled_by_provisioned_write_percent))
return throttled_by_provisioned_write_percent
def get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start=15, lookback_period=5):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: float -- Number of consumed writes as a
percentage of provisioned writes
"""
try:
metrics = __get_aws_metric(
table_name,
gsi_name,
lookback_window_start,
lookback_period,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
lookback_seconds = lookback_period * 60
consumed_write_units = (
float(metrics[0]['Sum']) / float(lookback_seconds))
else:
consumed_write_units = 0
try:
gsi_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = (
float(consumed_write_units) /
float(gsi_write_units) * 100)
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed write units: {2:.2f}%'.format(
table_name, gsi_name, consumed_write_units_percent))
return consumed_write_units_percent
def get_throttled_by_provisioned_write_event_percent(logger,
table_name,
gsi_name,
lookback_window_start=15,
lookback_period=5):
""" Returns the number of throttled write events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: float -- Percent of throttled write events by provisioning
"""
try:
metrics = __get_aws_metric(logger, table_name, gsi_name,
lookback_window_start, lookback_period,
'WriteThrottleEvents')
except BotoServerError:
raise
if metrics:
lookback_seconds = lookback_period * 60
throttled_write_events = float(
metrics[0]['Sum']) / float(lookback_seconds)
else:
throttled_write_events = 0
try:
gsi_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
throttled_by_provisioned_write_percent = (
float(throttled_write_events) / float(gsi_write_units) * 100)
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Throttled write percent '
'by provision: {2:.2f}%'.format(
table_name, gsi_name,
throttled_by_provisioned_write_percent))
return throttled_by_provisioned_write_percent
def get_consumed_write_units_percent(logger,
table_name,
gsi_name,
lookback_window_start=15,
lookback_period=5):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: float -- Number of consumed writes as a
percentage of provisioned writes
"""
try:
metrics = __get_aws_metric(logger, table_name, gsi_name,
lookback_window_start, lookback_period,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
lookback_seconds = lookback_period * 60
consumed_write_units = (float(metrics[0]['Sum']) /
float(lookback_seconds))
else:
consumed_write_units = 0
try:
gsi_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = (float(consumed_write_units) /
float(gsi_write_units) * 100)
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed write units: {2:.2f}%'.format(
table_name, gsi_name, consumed_write_units_percent))
return consumed_write_units_percent
def __ensure_provisioning_writes(
table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_write_units, num_consec_write_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of writes has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name), 0
update_needed = False
try:
current_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = \
gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
throttled_write_count = \
gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
writes_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
writes_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
throttled_writes_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_writes_upper_threshold')
increase_writes_unit = \
get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
increase_writes_with = \
get_gsi_option(table_key, gsi_key, 'increase_writes_with')
decrease_writes_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
decrease_writes_with = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
max_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
num_write_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_write_checks_before_scale_down')
num_write_checks_reset_percent = \
get_gsi_option(key_name, 'num_write_checks_reset_percent')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current write unit value
updated_write_units = current_write_units
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down writes is not done when usage is at 0%'.format(
table_name, gsi_name))
elif consumed_write_units_percent >= writes_upper_threshold:
if increase_writes_unit == 'percent':
calulated_provisioning = calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calulated_provisioning = calculators.increase_writes_in_units(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calulated_provisioning:
logger.info(
'{0} - Resetting the number of consecutive '
'write checks. Reason: scale up event detected'.format(
table_name))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calulated_provisioning
elif throttled_write_count > throttled_writes_upper_threshold:
if throttled_writes_upper_threshold > 0:
if increase_writes_unit == 'percent':
calulated_provisioning = calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calulated_provisioning = calculators.increase_writes_in_units(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calulated_provisioning:
logger.info(
'{0} - Resetting the number of consecutive '
'write checks. Reason: scale up event detected'.format(
table_name))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calulated_provisioning
elif consumed_write_units_percent >= num_write_checks_reset_percent:
logger.info(
'{0} - Resetting the number of consecutive '
'write checks. Reason: Consumed Percent {1} is Greater than Reset Percent: {2}'.format(
table_name, consumed_write_units_percent, num_write_checks_reset_percent))
num_consec_write_checks = 0
elif consumed_write_units_percent <= writes_lower_threshold:
if decrease_writes_unit == 'percent':
calulated_provisioning = calculators.decrease_writes_in_percent(
current_write_units,
decrease_writes_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calulated_provisioning = calculators.decrease_writes_in_units(
current_write_units,
decrease_writes_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calulated_provisioning:
num_consec_write_checks = num_consec_write_checks + 1
if num_consec_write_checks >= num_write_checks_before_scale_down:
update_needed = True
updated_write_units = calulated_provisioning
if max_provisioned_writes:
if int(updated_write_units) > int(max_provisioned_writes):
update_needed = True
updated_write_units = int(max_provisioned_writes)
logger.info(
'{0} - GSI: {1} - '
'Will not increase writes over gsi-max-provisioned-writes '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_write_units))
logger.info('{0} - Consecutive write checks {1}/{2}'.format(
table_name,
num_consec_write_checks,
num_write_checks_before_scale_down))
return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:returns: (bool, int) -- update_needed, updated_write_units
"""
if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of writes has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
update_needed = False
try:
updated_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = \
gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
throttled_write_count = \
gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
writes_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
writes_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
throttled_writes_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_writes_upper_threshold')
increase_writes_unit = \
get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
increase_writes_with = \
get_gsi_option(table_key, gsi_key, 'increase_writes_with')
decrease_writes_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
decrease_writes_with = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
max_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
except JSONResponseError:
raise
except BotoServerError:
raise
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down writes is not done when usage is at 0%'.format(
table_name, gsi_name))
elif consumed_write_units_percent >= writes_upper_threshold:
if increase_writes_unit == 'percent':
updated_provisioning = calculators.increase_writes_in_percent(
updated_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
updated_provisioning = calculators.increase_writes_in_units(
updated_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if updated_write_units != updated_provisioning:
update_needed = True
updated_write_units = updated_provisioning
elif throttled_write_count > throttled_writes_upper_threshold:
if throttled_writes_upper_threshold > 0:
if increase_writes_unit == 'percent':
updated_provisioning = calculators.increase_writes_in_percent(
updated_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
updated_provisioning = calculators.increase_writes_in_units(
updated_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if updated_write_units != updated_provisioning:
update_needed = True
updated_write_units = updated_provisioning
elif consumed_write_units_percent <= writes_lower_threshold:
if decrease_writes_unit == 'percent':
updated_provisioning = calculators.decrease_writes_in_percent(
updated_write_units,
decrease_writes_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
updated_provisioning = calculators.decrease_writes_in_units(
updated_write_units,
decrease_writes_with,
get_gsi_option(table_key, gsi_key, 'min_provisioned_reads'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if updated_write_units != updated_provisioning:
update_needed = True
updated_write_units = updated_provisioning
if max_provisioned_writes:
if int(updated_write_units) > int(max_provisioned_writes):
update_needed = True
updated_write_units = int(max_provisioned_writes)
logger.info(
'{0} - GSI: {1} - '
'Will not increase writes over gsi-max-provisioned-writes '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_write_units))
return update_needed, int(updated_write_units)
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key,
num_consec_write_checks):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_write_units, num_consec_write_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
logger.info('{0} - GSI: {1} - '
'Autoscaling of writes has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name), 0
update_needed = False
try:
lookback_window_start = get_gsi_option(table_key, gsi_key,
'lookback_window_start')
current_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = \
gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
throttled_write_count = \
gsi_stats.get_throttled_write_event_count(
table_name, gsi_name, lookback_window_start)
writes_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
writes_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
throttled_writes_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_writes_upper_threshold')
increase_writes_unit = \
get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
increase_writes_with = \
get_gsi_option(table_key, gsi_key, 'increase_writes_with')
decrease_writes_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
decrease_writes_with = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
max_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
num_write_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_write_checks_before_scale_down')
num_write_checks_reset_percent = \
get_gsi_option(table_key, gsi_key, 'num_write_checks_reset_percent')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current write unit value
updated_write_units = current_write_units
# Reset write consecutive count if num_write_checks_reset_percent is reached
if num_write_checks_reset_percent:
if consumed_write_units_percent >= num_write_checks_reset_percent:
logger.info('{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: Consumed percent {2} is '
'greater than reset percent: {3}'.format(
table_name, gsi_name, consumed_write_units_percent,
num_write_checks_reset_percent))
num_consec_write_checks = 0
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down writes is not done when usage is at 0%'.format(
table_name, gsi_name))
# Increase needed due to high CU consumption
elif consumed_write_units_percent >= writes_upper_threshold:
# Exit if up scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_writes_up_scaling'):
logger.debug(
'{0} - GSI: {1} - Up scaling event detected. '
'No action taken as scaling '
'up writes has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if increase_writes_unit == 'percent':
calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_writes_in_units(
current_write_units, increase_writes_with,
get_gsi_option(table_key, gsi_key,
'max_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: scale up event detected'.format(
table_name, gsi_name))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
# Increase needed due to high throttling
elif throttled_write_count > throttled_writes_upper_threshold:
if throttled_writes_upper_threshold > 0:
if increase_writes_unit == 'percent':
calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_writes_in_units(
current_write_units, increase_writes_with,
get_gsi_option(table_key, gsi_key,
'max_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: scale up event detected'.format(
table_name, gsi_name))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
# Decrease needed due to low CU consumption
elif consumed_write_units_percent <= writes_lower_threshold:
# Exit if down scaling has been disabled
if not get_gsi_option(table_key, gsi_key,
'enable_writes_down_scaling'):
logger.debug(
'{0} - GSI: {1} - Down scaling event detected. '
'No action taken as scaling '
'down writes has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if decrease_writes_unit == 'percent':
calculated_provisioning = \
calculators.decrease_writes_in_percent(
current_write_units,
decrease_writes_with,
get_gsi_option(
table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.decrease_writes_in_units(
current_write_units, decrease_writes_with,
get_gsi_option(table_key, gsi_key,
'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
num_consec_write_checks = num_consec_write_checks + 1
if (num_consec_write_checks >=
num_write_checks_before_scale_down):
update_needed = True
updated_write_units = calculated_provisioning
# Never go over the configured max provisioning
if max_provisioned_writes:
if int(updated_write_units) > int(max_provisioned_writes):
update_needed = True
updated_write_units = int(max_provisioned_writes)
logger.info(
'{0} - GSI: {1} - '
'Will not increase writes over gsi-max-provisioned-writes '
'limit ({2} writes)'.format(table_name, gsi_name,
updated_write_units))
logger.info('{0} - GSI: {1} - Consecutive write checks {2}/{3}'.format(
table_name, gsi_name, num_consec_write_checks,
num_write_checks_before_scale_down))
return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_write_units, num_consec_write_checks
"""
if not get_gsi_option(table_key, gsi_key, "enable_writes_autoscaling"):
logger.info("{0} - GSI: {1} - " "Autoscaling of writes has been disabled".format(table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name), 0
update_needed = False
try:
current_write_units = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
throttled_write_count = gsi_stats.get_throttled_write_event_count(table_name, gsi_name)
writes_upper_threshold = get_gsi_option(table_key, gsi_key, "writes_upper_threshold")
writes_lower_threshold = get_gsi_option(table_key, gsi_key, "writes_lower_threshold")
throttled_writes_upper_threshold = get_gsi_option(table_key, gsi_key, "throttled_writes_upper_threshold")
increase_writes_unit = get_gsi_option(table_key, gsi_key, "increase_writes_unit")
increase_writes_with = get_gsi_option(table_key, gsi_key, "increase_writes_with")
decrease_writes_unit = get_gsi_option(table_key, gsi_key, "decrease_writes_unit")
decrease_writes_with = get_gsi_option(table_key, gsi_key, "decrease_writes_with")
max_provisioned_writes = get_gsi_option(table_key, gsi_key, "max_provisioned_writes")
num_write_checks_before_scale_down = get_gsi_option(table_key, gsi_key, "num_write_checks_before_scale_down")
num_write_checks_reset_percent = get_gsi_option(table_key, gsi_key, "num_write_checks_reset_percent")
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current write unit value
updated_write_units = current_write_units
# Reset write consecutive count if num_write_checks_reset_percent is reached
if num_write_checks_reset_percent:
if consumed_write_units_percent >= num_write_checks_reset_percent:
logger.info(
"{0} - GSI: {1} - Resetting the number of consecutive "
"write checks. Reason: Consumed percent {2} is "
"greater than reset percent: {3}".format(
table_name, gsi_name, consumed_write_units_percent, num_write_checks_reset_percent
)
)
num_consec_write_checks = 0
# Check if we should update write provisioning
if consumed_write_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, "allow_scaling_down_writes_on_0_percent"
):
logger.info(
"{0} - GSI: {1} - " "Scaling down writes is not done when usage is at 0%".format(table_name, gsi_name)
)
elif consumed_write_units_percent >= writes_upper_threshold:
if increase_writes_unit == "percent":
calculated_provisioning = calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
else:
calculated_provisioning = calculators.increase_writes_in_units(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
if current_write_units != calculated_provisioning:
logger.info(
"{0} - GSI: {1} - Resetting the number of consecutive "
"write checks. Reason: scale up event detected".format(table_name, gsi_name)
)
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
elif throttled_write_count > throttled_writes_upper_threshold:
if throttled_writes_upper_threshold > 0:
if increase_writes_unit == "percent":
calculated_provisioning = calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
else:
calculated_provisioning = calculators.increase_writes_in_units(
current_write_units,
increase_writes_with,
get_gsi_option(table_key, gsi_key, "max_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
if current_write_units != calculated_provisioning:
logger.info(
"{0} - GSI: {1} - Resetting the number of consecutive "
"write checks. Reason: scale up event detected".format(table_name, gsi_name)
)
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
elif consumed_write_units_percent <= writes_lower_threshold:
if decrease_writes_unit == "percent":
calculated_provisioning = calculators.decrease_writes_in_percent(
current_write_units,
decrease_writes_with,
get_gsi_option(table_key, gsi_key, "min_provisioned_writes"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
else:
calculated_provisioning = calculators.decrease_writes_in_units(
current_write_units,
decrease_writes_with,
get_gsi_option(table_key, gsi_key, "min_provisioned_reads"),
"{0} - GSI: {1}".format(table_name, gsi_name),
)
if current_write_units != calculated_provisioning:
num_consec_write_checks = num_consec_write_checks + 1
if num_consec_write_checks >= num_write_checks_before_scale_down:
update_needed = True
updated_write_units = calculated_provisioning
if max_provisioned_writes:
if int(updated_write_units) > int(max_provisioned_writes):
update_needed = True
updated_write_units = int(max_provisioned_writes)
logger.info(
"{0} - GSI: {1} - "
"Will not increase writes over gsi-max-provisioned-writes "
"limit ({2} writes)".format(table_name, gsi_name, updated_write_units)
)
logger.info(
"{0} - GSI: {1} - Consecutive write checks {2}/{3}".format(
table_name, gsi_name, num_consec_write_checks, num_write_checks_before_scale_down
)
)
return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(
table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_write_units, num_consec_write_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of writes has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name), 0
update_needed = False
try:
lookback_window_start = get_gsi_option(
table_key, gsi_key, 'lookback_window_start')
current_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = \
gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
throttled_write_count = \
gsi_stats.get_throttled_write_event_count(
table_name, gsi_name, lookback_window_start)
writes_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
writes_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
throttled_writes_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_writes_upper_threshold')
increase_writes_unit = \
get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
increase_writes_with = \
get_gsi_option(table_key, gsi_key, 'increase_writes_with')
decrease_writes_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
decrease_writes_with = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
min_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'min_provisioned_writes')
max_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
num_write_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_write_checks_before_scale_down')
num_write_checks_reset_percent = \
get_gsi_option(table_key, gsi_key, 'num_write_checks_reset_percent')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current write unit value
updated_write_units = current_write_units
# Reset write consecutive count if num_write_checks_reset_percent is reached
if num_write_checks_reset_percent:
if consumed_write_units_percent >= num_write_checks_reset_percent:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: Consumed percent {2} is '
'greater than reset percent: {3}'.format(
table_name,
gsi_name,
consumed_write_units_percent,
num_write_checks_reset_percent))
num_consec_write_checks = 0
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down writes is not done when usage is at 0%'.format(
table_name, gsi_name))
# Increase needed due to high CU consumption
elif consumed_write_units_percent >= writes_upper_threshold:
# Exit if up scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_writes_up_scaling'):
logger.debug(
'{0} - GSI: {1} - Up scaling event detected. '
'No action taken as scaling '
'up writes has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if increase_writes_unit == 'percent':
calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_writes_in_units(
current_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: scale up event detected'.format(
table_name, gsi_name))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
# Increase needed due to high throttling
elif throttled_write_count > throttled_writes_upper_threshold:
if throttled_writes_upper_threshold > 0:
if increase_writes_unit == 'percent':
calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.increase_writes_in_units(
current_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: scale up event detected'.format(
table_name, gsi_name))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
# Decrease needed due to low CU consumption
elif consumed_write_units_percent <= writes_lower_threshold:
# Exit if down scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_writes_down_scaling'):
logger.debug(
'{0} - GSI: {1} - Down scaling event detected. '
'No action taken as scaling '
'down writes has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if decrease_writes_unit == 'percent':
calculated_provisioning = \
calculators.decrease_writes_in_percent(
current_write_units,
decrease_writes_with,
get_gsi_option(
table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.decrease_writes_in_units(
current_write_units,
decrease_writes_with,
get_gsi_option(
table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
num_consec_write_checks = num_consec_write_checks + 1
if (num_consec_write_checks >=
num_write_checks_before_scale_down):
update_needed = True
updated_write_units = calculated_provisioning
# Never go over the configured max provisioning
if max_provisioned_writes:
if int(updated_write_units) > int(max_provisioned_writes):
update_needed = True
updated_write_units = int(max_provisioned_writes)
logger.info(
'{0} - GSI: {1} - '
'Will not increase writes over gsi-max-provisioned-writes '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_write_units))
# Ensure that we have met the min-provisioning
if min_provisioned_writes:
if int(min_provisioned_writes) > int(updated_write_units):
update_needed = True
updated_write_units = int(min_provisioned_writes)
logger.info(
'{0} - GSI: {1} - Increasing writes to'
'meet gsi-min-provisioned-writes '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_write_units))
logger.info('{0} - GSI: {1} - Consecutive write checks {2}/{3}'.format(
table_name,
gsi_name,
num_consec_write_checks,
num_write_checks_before_scale_down))
return update_needed, updated_write_units, num_consec_write_checks
def __ensure_provisioning_writes(
table_name, table_key, gsi_name, gsi_key, num_consec_write_checks):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (bool, int, int)
update_needed, updated_write_units, num_consec_write_checks
"""
if not get_gsi_option(table_key, gsi_key, 'enable_writes_autoscaling'):
logger.info(
'{0} - GSI: {1} - '
'Autoscaling of writes has been disabled'.format(
table_name, gsi_name))
return False, dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name), 0
update_needed = False
try:
lookback_window_start = get_gsi_option(
table_key, gsi_key, 'lookback_window_start')
current_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = \
gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
throttled_write_count = \
gsi_stats.get_throttled_write_event_count(
table_name, gsi_name, lookback_window_start)
throttled_by_provisioned_write_percent = \
gsi_stats.get_throttled_by_provisioned_write_event_percent(
table_name, gsi_name, lookback_window_start)
throttled_by_consumed_write_percent = \
gsi_stats.get_throttled_by_consumed_write_percent(
table_name, gsi_name, lookback_window_start)
writes_upper_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_upper_threshold')
writes_lower_threshold = \
get_gsi_option(table_key, gsi_key, 'writes_lower_threshold')
throttled_writes_upper_threshold = \
get_gsi_option(
table_key, gsi_key, 'throttled_writes_upper_threshold')
increase_writes_unit = \
get_gsi_option(table_key, gsi_key, 'increase_writes_unit')
increase_writes_with = \
get_gsi_option(table_key, gsi_key, 'increase_writes_with')
decrease_writes_unit = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_unit')
decrease_writes_with = \
get_gsi_option(table_key, gsi_key, 'decrease_writes_with')
min_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'min_provisioned_writes')
max_provisioned_writes = \
get_gsi_option(table_key, gsi_key, 'max_provisioned_writes')
num_write_checks_before_scale_down = \
get_gsi_option(
table_key, gsi_key, 'num_write_checks_before_scale_down')
num_write_checks_reset_percent = \
get_gsi_option(table_key, gsi_key, 'num_write_checks_reset_percent')
increase_throttled_by_provisioned_writes_unit = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_provisioned_writes_unit')
increase_throttled_by_provisioned_writes_scale = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_provisioned_writes_scale')
increase_throttled_by_consumed_writes_unit = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_consumed_writes_unit')
increase_throttled_by_consumed_writes_scale = \
get_gsi_option(
table_key,
gsi_key,
'increase_throttled_by_consumed_writes_scale')
increase_consumed_writes_unit = \
get_gsi_option(table_key, gsi_key, 'increase_consumed_writes_unit')
increase_consumed_writes_with = \
get_gsi_option(table_key, gsi_key, 'increase_consumed_writes_with')
increase_consumed_writes_scale = \
get_gsi_option(table_key, gsi_key, 'increase_consumed_writes_scale')
except JSONResponseError:
raise
except BotoServerError:
raise
# Set the updated units to the current write unit value
updated_write_units = current_write_units
# Reset write consecutive count if num_write_checks_reset_percent is reached
if num_write_checks_reset_percent:
if consumed_write_units_percent >= num_write_checks_reset_percent:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: Consumed percent {2} is '
'greater than reset percent: {3}'.format(
table_name,
gsi_name,
consumed_write_units_percent,
num_write_checks_reset_percent))
num_consec_write_checks = 0
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not get_gsi_option(
table_key, gsi_key, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - GSI: {1} - '
'Scaling down writes is not done when usage is at 0%'.format(
table_name, gsi_name))
# Exit if up scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_writes_up_scaling'):
logger.debug(
'{0} - GSI: {1} - Up scaling event detected. No action taken as '
'scaling up writes has been disabled in the configuration'.format(
table_name, gsi_name))
else:
# If local/granular values not specified use global values
increase_consumed_writes_unit = \
increase_consumed_writes_unit or increase_writes_unit
increase_throttled_by_provisioned_writes_unit = (
increase_throttled_by_provisioned_writes_unit
or increase_writes_unit)
increase_throttled_by_consumed_writes_unit = \
increase_throttled_by_consumed_writes_unit or increase_writes_unit
increase_consumed_writes_with = \
increase_consumed_writes_with or increase_writes_with
# Initialise variables to store calculated provisioning
throttled_by_provisioned_calculated_provisioning = scale_reader(
increase_throttled_by_provisioned_writes_scale,
throttled_by_provisioned_write_percent)
throttled_by_consumed_calculated_provisioning = scale_reader(
increase_throttled_by_consumed_writes_scale,
throttled_by_consumed_write_percent)
consumed_calculated_provisioning = scale_reader(
increase_consumed_writes_scale,
consumed_write_units_percent)
throttled_count_calculated_provisioning = 0
calculated_provisioning = 0
# Increase needed due to high throttled to provisioned ratio
if throttled_by_provisioned_calculated_provisioning:
if increase_throttled_by_provisioned_writes_unit == 'percent':
throttled_by_provisioned_calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
throttled_by_provisioned_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
throttled_by_provisioned_calculated_provisioning = \
calculators.increase_writes_in_units(
current_write_units,
throttled_by_provisioned_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Increase needed due to high throttled to consumed ratio
if throttled_by_consumed_calculated_provisioning:
if increase_throttled_by_consumed_writes_unit == 'percent':
throttled_by_consumed_calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
throttled_by_consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
throttled_by_consumed_calculated_provisioning = \
calculators.increase_writes_in_units(
current_write_units,
throttled_by_consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Increase needed due to high CU consumption
if consumed_calculated_provisioning:
if increase_consumed_writes_unit == 'percent':
consumed_calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
consumed_calculated_provisioning = \
calculators.increase_writes_in_units(
current_write_units,
consumed_calculated_provisioning,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
elif (writes_upper_threshold
and consumed_write_units_percent > writes_upper_threshold
and not increase_consumed_writes_scale):
if increase_consumed_writes_unit == 'percent':
consumed_calculated_provisioning = \
calculators.increase_writes_in_percent(
current_write_units,
increase_consumed_writes_with,
get_gsi_option(
table_key,
gsi_key,
'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
consumed_calculated_provisioning = \
calculators.increase_writes_in_units(
current_write_units,
increase_consumed_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Increase needed due to high throttling
if (throttled_writes_upper_threshold
and throttled_write_count > throttled_writes_upper_threshold):
if increase_writes_unit == 'percent':
throttled_count_calculated_provisioning = \
calculators.increase_writes_in_percent(
updated_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
throttled_count_calculated_provisioning = \
calculators.increase_writes_in_units(
updated_write_units,
increase_writes_with,
get_gsi_option(
table_key, gsi_key, 'max_provisioned_writes'),
consumed_write_units_percent,
'{0} - GSI: {1}'.format(table_name, gsi_name))
# Determine which metric requires the most scaling
if (throttled_by_provisioned_calculated_provisioning
> calculated_provisioning):
calculated_provisioning = \
throttled_by_provisioned_calculated_provisioning
scale_reason = (
"due to throttled events by provisioned "
"units threshold being exceeded")
if (throttled_by_consumed_calculated_provisioning
> calculated_provisioning):
calculated_provisioning = \
throttled_by_consumed_calculated_provisioning
scale_reason = (
"due to throttled events by consumed "
"units threshold being exceeded")
if consumed_calculated_provisioning > calculated_provisioning:
calculated_provisioning = consumed_calculated_provisioning
scale_reason = "due to consumed threshold being exceeded"
if throttled_count_calculated_provisioning > calculated_provisioning:
calculated_provisioning = throttled_count_calculated_provisioning
scale_reason = "due to throttled events threshold being exceeded"
if calculated_provisioning > current_write_units:
logger.info(
'{0} - GSI: {1} - Resetting the number of consecutive '
'write checks. Reason: scale up {2}'.format(
table_name, gsi_name, scale_reason))
num_consec_write_checks = 0
update_needed = True
updated_write_units = calculated_provisioning
# Decrease needed due to low CU consumption
if (consumed_write_units_percent
<= writes_lower_threshold and not update_needed):
# Exit if down scaling has been disabled
if not get_gsi_option(table_key, gsi_key, 'enable_writes_down_scaling'):
logger.debug(
'{0} - GSI: {1} - Down scaling event detected. '
'No action taken as scaling '
'down writes has been disabled in the configuration'.format(
table_name, gsi_name))
else:
if decrease_writes_unit == 'percent':
calculated_provisioning = \
calculators.decrease_writes_in_percent(
current_write_units,
decrease_writes_with,
get_gsi_option(
table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
else:
calculated_provisioning = calculators.decrease_writes_in_units(
current_write_units,
decrease_writes_with,
get_gsi_option(
table_key, gsi_key, 'min_provisioned_writes'),
'{0} - GSI: {1}'.format(table_name, gsi_name))
if current_write_units != calculated_provisioning:
num_consec_write_checks = num_consec_write_checks + 1
if (num_consec_write_checks >=
num_write_checks_before_scale_down):
update_needed = True
updated_write_units = calculated_provisioning
# Never go over the configured max provisioning
if max_provisioned_writes:
if int(updated_write_units) > int(max_provisioned_writes):
update_needed = True
updated_write_units = int(max_provisioned_writes)
logger.info(
'{0} - GSI: {1} - '
'Will not increase writes over gsi-max-provisioned-writes '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_write_units))
# Ensure that we have met the min-provisioning
if min_provisioned_writes:
if int(min_provisioned_writes) > int(updated_write_units):
update_needed = True
updated_write_units = int(min_provisioned_writes)
logger.info(
'{0} - GSI: {1} - Increasing writes to'
'meet gsi-min-provisioned-writes '
'limit ({2} writes)'.format(
table_name,
gsi_name,
updated_write_units))
logger.info('{0} - GSI: {1} - Consecutive write checks {2}/{3}'.format(
table_name,
gsi_name,
num_consec_write_checks,
num_write_checks_before_scale_down))
return update_needed, updated_write_units, num_consec_write_checks
