def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
"""
lookback_window_start = get_gsi_option(
table_key, gsi_key, 'lookback_window_start')
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
reads_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold')
reads_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold')
writes_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold')
writes_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold')
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if (reads_upper_alarm_threshold > 0 and
consumed_read_units_percent >= reads_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_read_units_percent,
reads_upper_alarm_threshold))
if (writes_upper_alarm_threshold > 0 and
consumed_write_units_percent >= writes_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_write_units_percent,
writes_upper_alarm_threshold))
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if (reads_lower_alarm_threshold > 0 and
consumed_read_units_percent < reads_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_read_units_percent,
reads_lower_alarm_threshold))
if (writes_lower_alarm_threshold > 0 and
consumed_write_units_percent < writes_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name,
gsi_name,
consumed_write_units_percent,
writes_lower_alarm_threshold))
# Send alert if needed
if upper_alert_triggered:
logger.info(
'{0} - GSI: {1} - Will send high provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(upper_alert_message),
['high-throughput-alarm'],
subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
elif lower_alert_triggered:
logger.info(
'{0} - GSI: {1} - Will send low provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(lower_alert_message),
['low-throughput-alarm'],
subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
else:
logger.debug(
'{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format(
table_name, gsi_name))
def update_gsi_provisioning(table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
# Check that we are in the right time frame
m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows')
if m_windows:
if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
logger.warning(
'{0} - GSI: {1} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(
table_name, gsi_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info('{0} - GSI: {1} - '
'No need to scale up reads nor writes'.format(
table_name, gsi_name))
return
else:
logger.info('{0} - GSI: {1} - '
'Current time is within maintenance window'.format(
table_name, gsi_name))
logger.info('{0} - GSI: {1} - '
'Updating provisioning to {2} reads and {3} writes'.format(
table_name, gsi_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
DYNAMODB_CONNECTION.update_table(table_name=table_name,
global_secondary_index_updates=[{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits":
reads,
"WriteCapacityUnits":
writes
}
}
}])
message = ('{0} - GSI: {1} - Provisioning updated to '
'{2} reads and {3} writes').format(table_name, gsi_name,
reads, writes)
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < current_writes:
sns_message_types.append('scale-up')
sns.publish_gsi_notification(
table_key,
gsi_key,
message,
sns_message_types,
subject='Updated provisioning for GSI {0}'.format(gsi_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
('{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues').format(
table_name, gsi_name, exception, error.body['message']))
if (not retry_with_only_increase
and exception == 'LimitExceededException'):
logger.info('{0} - GSI: {1} - Will retry to update provisioning '
'with only increases'.format(table_name, gsi_name))
update_gsi_provisioning(table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=True)
def update_gsi_provisioning(
table_name, table_key, gsi_name, gsi_key,
reads, writes, retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name))
current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name))
# Make sure we aren't scaling down if we turned off downscaling
if (not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling') or
not get_gsi_option(
table_key, gsi_key, 'enable_writes_down_scaling')):
if (not get_gsi_option(
table_key, gsi_key, 'enable_reads_down_scaling') and
current_reads > reads):
reads = current_reads
if (not get_gsi_option(
table_key, gsi_key, 'enable_writes_down_scaling') and
current_writes > writes):
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - GSI: {1} - No need to scale up reads nor writes'.format(
table_name, gsi_name))
return
logger.info(
'{0} - GSI: {1} - Retrying to update provisioning, '
'excluding any decreases. '
'Setting new reads to {2} and new writes to {3}'.format(
table_name, gsi_name, reads, writes))
# Check that we are in the right time frame
m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows')
if m_windows:
if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
logger.warning(
'{0} - GSI: {1} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(
table_name,
gsi_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - GSI: {1} - '
'No need to scale up reads nor writes'.format(
table_name,
gsi_name))
return
else:
logger.info(
'{0} - GSI: {1} - '
'Current time is within maintenance window'.format(
table_name,
gsi_name))
logger.info(
'{0} - GSI: {1} - '
'Updating provisioning to {2} reads and {3} writes'.format(
table_name, gsi_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits": reads,
"WriteCapacityUnits": writes
}
}
}
])
message = []
if current_reads > reads:
message.append(
'{0} - GSI: {1} - Reads: DOWN from {2} to {3}\n'.format(
table_name, gsi_name, current_reads, reads))
elif current_reads < reads:
message.append(
'{0} - GSI: {1} - Reads: UP from {2} to {3}\n'.format(
table_name, gsi_name, current_reads, reads))
if current_writes > writes:
message.append(
'{0} - GSI: {1} - Writes: DOWN from {2} to {3}\n'.format(
table_name, gsi_name, current_writes, writes))
elif current_writes < writes:
message.append(
'{0} - GSI: {1} - Writes: UP from {2} to {3}\n'.format(
table_name, gsi_name, current_writes, writes))
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < writes:
sns_message_types.append('scale-up')
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(message),
sns_message_types,
subject='Updated provisioning for GSI {0}'.format(gsi_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name, gsi_name, exception, error.body['message']))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - GSI: {1} - Will retry to update provisioning '
'with only increases'.format(table_name, gsi_name))
update_gsi_provisioning(
table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=True)
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
"""
lookback_window_start = get_gsi_option(table_key, gsi_key,
'lookback_window_start')
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(
table_name, gsi_name, lookback_window_start)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(
table_name, gsi_name, lookback_window_start)
reads_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold')
reads_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold')
writes_upper_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold')
writes_lower_alarm_threshold = \
get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold')
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if (reads_upper_alarm_threshold > 0
and consumed_read_units_percent >= reads_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(table_name, gsi_name,
consumed_read_units_percent,
reads_upper_alarm_threshold))
if (writes_upper_alarm_threshold > 0
and consumed_write_units_percent >= writes_upper_alarm_threshold):
upper_alert_triggered = True
upper_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was greater than or equal to the upper alarm '
'threshold {3:d}%\n'.format(table_name, gsi_name,
consumed_write_units_percent,
writes_upper_alarm_threshold))
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if (reads_lower_alarm_threshold > 0
and consumed_read_units_percent < reads_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Read Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name, gsi_name, consumed_read_units_percent,
reads_lower_alarm_threshold))
if (writes_lower_alarm_threshold > 0
and consumed_write_units_percent < writes_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - GSI: {1} - Consumed Write Capacity {2:d}% '
'was below the lower alarm threshold {3:d}%\n'.format(
table_name, gsi_name, consumed_write_units_percent,
writes_lower_alarm_threshold))
# Send alert if needed
if upper_alert_triggered:
logger.info(
'{0} - GSI: {1} - Will send high provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(upper_alert_message), ['high-throughput-alarm'],
subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
elif lower_alert_triggered:
logger.info('{0} - GSI: {1} - Will send low provisioning alert'.format(
table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(lower_alert_message), ['low-throughput-alarm'],
subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format(
table_name, gsi_name))
else:
logger.debug(
'{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format(
table_name, gsi_name))
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
"""
consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name)
consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name)
reads_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-upper-alarm-threshold")
reads_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-lower-alarm-threshold")
writes_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-upper-alarm-threshold")
writes_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-lower-alarm-threshold")
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if reads_upper_alarm_threshold > 0 and consumed_read_units_percent >= reads_upper_alarm_threshold:
upper_alert_triggered = True
upper_alert_message.append(
"{0} - GSI: {1} - Consumed Read Capacity {2:d}% "
"was greater than or equal to the upper alarm "
"threshold {3:d}%\n".format(table_name, gsi_name, consumed_read_units_percent, reads_upper_alarm_threshold)
)
if writes_upper_alarm_threshold > 0 and consumed_write_units_percent >= writes_upper_alarm_threshold:
upper_alert_triggered = True
upper_alert_message.append(
"{0} - GSI: {1} - Consumed Write Capacity {2:d}% "
"was greater than or equal to the upper alarm "
"threshold {3:d}%\n".format(
table_name, gsi_name, consumed_write_units_percent, writes_upper_alarm_threshold
)
)
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if reads_lower_alarm_threshold > 0 and consumed_read_units_percent < reads_lower_alarm_threshold:
lower_alert_triggered = True
lower_alert_message.append(
"{0} - GSI: {1} - Consumed Read Capacity {2:d}% "
"was below the lower alarm threshold {3:d}%\n".format(
table_name, gsi_name, consumed_read_units_percent, reads_lower_alarm_threshold
)
)
if writes_lower_alarm_threshold > 0 and consumed_write_units_percent < writes_lower_alarm_threshold:
lower_alert_triggered = True
lower_alert_message.append(
"{0} - GSI: {1} - Consumed Write Capacity {2:d}% "
"was below the lower alarm threshold {3:d}%\n".format(
table_name, gsi_name, consumed_write_units_percent, writes_lower_alarm_threshold
)
)
# Send alert if needed
if upper_alert_triggered:
logger.info("{0} - GSI: {1} - Will send high provisioning alert".format(table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
"".join(upper_alert_message),
["high-throughput-alarm"],
subject="ALARM: High Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name),
)
elif lower_alert_triggered:
logger.info("{0} - GSI: {1} - Will send low provisioning alert".format(table_name, gsi_name))
sns.publish_gsi_notification(
table_key,
gsi_key,
"".join(lower_alert_message),
["low-throughput-alarm"],
subject="ALARM: Low Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name),
)
else:
logger.debug("{0} - GSI: {1} - Throughput alarm thresholds not crossed".format(table_name, gsi_name))
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name))
current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info("{0} - GSI: {1} - No need to scale up reads nor writes".format(table_name, gsi_name))
return
logger.info(
"{0} - GSI: {1} - Retrying to update provisioning, excluding any decreases. "
"Setting new reads to {2} and new writes to {3}".format(table_name, gsi_name, reads, writes)
)
# Check that we are in the right time frame
m_windows = get_gsi_option(table_key, gsi_key, "maintenance_windows")
if m_windows:
if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
logger.warning(
"{0} - GSI: {1} - We are outside a maintenace window. "
"Will only perform up scaling activites".format(table_name, gsi_name)
)
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info("{0} - GSI: {1} - " "No need to scale up reads nor writes".format(table_name, gsi_name))
return
else:
logger.info("{0} - GSI: {1} - " "Current time is within maintenance window".format(table_name, gsi_name))
logger.info(
"{0} - GSI: {1} - "
"Updating provisioning to {2} reads and {3} writes".format(table_name, gsi_name, reads, writes)
)
# Return if dry-run
if get_global_option("dry_run"):
return
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {"ReadCapacityUnits": reads, "WriteCapacityUnits": writes},
}
}
],
)
message = ("{0} - GSI: {1} - Provisioning updated to " "{2} reads and {3} writes").format(
table_name, gsi_name, reads, writes
)
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append("scale-down")
if current_reads < reads or current_writes < current_writes:
sns_message_types.append("scale-up")
sns.publish_gsi_notification(
table_key, gsi_key, message, sns_message_types, subject="Updated provisioning for GSI {0}".format(gsi_name)
)
except JSONResponseError as error:
exception = error.body["__type"].split("#")[1]
know_exceptions = ["LimitExceededException"]
if exception in know_exceptions:
logger.warning("{0} - GSI: {1} - {2}: {3}".format(table_name, gsi_name, exception, error.body["message"]))
else:
logger.error(
(
"{0} - GSI: {1} - Unhandled exception: {2}: {3}. "
"Please file a bug report at "
"https://github.com/sebdah/dynamic-dynamodb/issues"
).format(table_name, gsi_name, exception, error.body["message"])
)
if not retry_with_only_increase and exception == "LimitExceededException":
logger.info(
"{0} - GSI: {1} - Will retry to update provisioning " "with only increases".format(table_name, gsi_name)
)
update_gsi_provisioning(
table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True
)