def __ensure_provisioning_alarm(table_name, key_name):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
"""
lookback_window_start = get_table_option(key_name, 'lookback_window_start')
lookback_period = get_table_option(key_name, 'lookback_period')
consumed_read_units_percent = table_stats.get_consumed_read_units_percent(
table_name, lookback_window_start, lookback_period)
consumed_write_units_percent = table_stats.get_consumed_write_units_percent(
table_name, lookback_window_start, lookback_period)
reads_upper_alarm_threshold = \
get_table_option(key_name, 'reads-upper-alarm-threshold')
reads_lower_alarm_threshold = \
get_table_option(key_name, 'reads-lower-alarm-threshold')
writes_upper_alarm_threshold = \
get_table_option(key_name, 'writes-upper-alarm-threshold')
writes_lower_alarm_threshold = \
get_table_option(key_name, 'writes-lower-alarm-threshold')
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if 0 < reads_upper_alarm_threshold <= consumed_read_units_percent:
upper_alert_triggered = True
upper_alert_message.append('{0} - Consumed Read Capacity {1:f}% '
'was greater than or equal to the upper '
'alarm threshold {2:f}%\n'.format(
table_name, consumed_read_units_percent,
reads_upper_alarm_threshold))
if 0 < writes_upper_alarm_threshold <= consumed_write_units_percent:
upper_alert_triggered = True
upper_alert_message.append(
'{0} - Consumed Write Capacity {1:f}% '
'was greater than or equal to the upper alarm '
'threshold {2:f}%\n'.format(table_name,
consumed_write_units_percent,
writes_upper_alarm_threshold))
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if (reads_lower_alarm_threshold > 0
and consumed_read_units_percent < reads_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - Consumed Read Capacity {1:f}% '
'was below the lower alarm threshold {2:f}%\n'.format(
table_name, consumed_read_units_percent,
reads_lower_alarm_threshold))
if (writes_lower_alarm_threshold > 0
and consumed_write_units_percent < writes_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - Consumed Write Capacity {1:f}% '
'was below the lower alarm threshold {2:f}%\n'.format(
table_name, consumed_write_units_percent,
writes_lower_alarm_threshold))
# Send alert if needed
if upper_alert_triggered:
logger.info(
'{0} - Will send high provisioning alert'.format(table_name))
sns.publish_table_notification(
key_name,
''.join(upper_alert_message), ['high-throughput-alarm'],
subject='ALARM: High Throughput for Table {0}'.format(table_name))
elif lower_alert_triggered:
logger.info(
'{0} - Will send low provisioning alert'.format(table_name))
sns.publish_table_notification(
key_name,
''.join(lower_alert_message), ['low-throughput-alarm'],
subject='ALARM: Low Throughput for Table {0}'.format(table_name))
else:
logger.debug(
'{0} - Throughput alarm thresholds not crossed'.format(table_name))
def update_table_provisioning(table_name,
key_name,
reads,
writes,
retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Configuration option key name
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
# Check that we are in the right time frame
maintenance_windows = get_table_option(key_name, 'maintenance_windows')
if maintenance_windows:
if not __is_table_maintenance_window(table_name, maintenance_windows):
logger.warning(
'{0} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(table_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
logger.info(
'{0} - Updating provisioning to {1} reads and {2} writes'.format(
table_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
table.update(throughput={'read': reads, 'write': writes})
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < current_writes:
sns_message_types.append('scale-up')
message = (
'{0} - Provisioning updated to {1} reads and {2} writes').format(
table_name, reads, writes)
sns.publish_table_notification(
key_name,
message,
sns_message_types,
subject='Updated provisioning for table {0}'.format(table_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException', 'ValidationException',
'ResourceInUseException'
]
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(table_name, exception,
error.body['message']))
else:
logger.error(
('{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues').format(
table_name, exception, error.body['message']))
if (not retry_with_only_increase
and exception == 'LimitExceededException'):
logger.info('{0} - Will retry to update provisioning '
'with only increases'.format(table_name))
update_table_provisioning(table_name,
key_name,
reads,
writes,
retry_with_only_increase=True)
def __ensure_provisioning_alarm(table_name, key_name):
""" Ensure that provisioning alarm threshold is not exceeded
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
"""
lookback_window_start = get_table_option(
key_name, 'lookback_window_start')
lookback_period = get_table_option(key_name, 'lookback_period')
consumed_read_units_percent = table_stats.get_consumed_read_units_percent(
table_name, lookback_window_start, lookback_period)
consumed_write_units_percent = table_stats.get_consumed_write_units_percent(
table_name, lookback_window_start, lookback_period)
reads_upper_alarm_threshold = \
get_table_option(key_name, 'reads-upper-alarm-threshold')
reads_lower_alarm_threshold = \
get_table_option(key_name, 'reads-lower-alarm-threshold')
writes_upper_alarm_threshold = \
get_table_option(key_name, 'writes-upper-alarm-threshold')
writes_lower_alarm_threshold = \
get_table_option(key_name, 'writes-lower-alarm-threshold')
# Check upper alarm thresholds
upper_alert_triggered = False
upper_alert_message = []
if 0 < reads_upper_alarm_threshold <= consumed_read_units_percent:
upper_alert_triggered = True
upper_alert_message.append(
'{0} - Consumed Read Capacity {1:f}% '
'was greater than or equal to the upper '
'alarm threshold {2:f}%\n'.format(
table_name,
consumed_read_units_percent,
reads_upper_alarm_threshold))
if 0 < writes_upper_alarm_threshold <= consumed_write_units_percent:
upper_alert_triggered = True
upper_alert_message.append(
'{0} - Consumed Write Capacity {1:f}% '
'was greater than or equal to the upper alarm '
'threshold {2:f}%\n'.format(
table_name,
consumed_write_units_percent,
writes_upper_alarm_threshold))
# Check lower alarm thresholds
lower_alert_triggered = False
lower_alert_message = []
if (reads_lower_alarm_threshold > 0 and
consumed_read_units_percent < reads_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - Consumed Read Capacity {1:f}% '
'was below the lower alarm threshold {2:f}%\n'.format(
table_name,
consumed_read_units_percent,
reads_lower_alarm_threshold))
if (writes_lower_alarm_threshold > 0 and
consumed_write_units_percent < writes_lower_alarm_threshold):
lower_alert_triggered = True
lower_alert_message.append(
'{0} - Consumed Write Capacity {1:f}% '
'was below the lower alarm threshold {2:f}%\n'.format(
table_name,
consumed_write_units_percent,
writes_lower_alarm_threshold))
# Send alert if needed
if upper_alert_triggered:
logger.info(
'{0} - Will send high provisioning alert'.format(table_name))
sns.publish_table_notification(
key_name,
''.join(upper_alert_message),
['high-throughput-alarm'],
subject='ALARM: High Throughput for Table {0}'.format(table_name))
elif lower_alert_triggered:
logger.info(
'{0} - Will send low provisioning alert'.format(table_name))
sns.publish_table_notification(
key_name,
''.join(lower_alert_message),
['low-throughput-alarm'],
subject='ALARM: Low Throughput for Table {0}'.format(table_name))
else:
logger.debug('{0} - Throughput alarm thresholds not crossed'.format(
table_name))
def update_table_provisioning(
table_name, key_name, reads, writes, retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Configuration option key name
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
# Make sure we aren't scaling down if we turned off downscaling
if (not get_table_option(key_name, 'enable_reads_down_scaling') or
not get_table_option(key_name, 'enable_writes_down_scaling')):
if (not get_table_option(key_name, 'enable_reads_down_scaling') and
current_reads > reads):
reads = current_reads
if (not get_table_option(key_name, 'enable_writes_down_scaling') and
current_writes > writes):
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
# Check that we are in the right time frame
maintenance_windows = get_table_option(key_name, 'maintenance_windows')
if maintenance_windows:
if not __is_table_maintenance_window(table_name, maintenance_windows):
logger.warning(
'{0} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(table_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
logger.info(
'{0} - Updating provisioning to {1} reads and {2} writes'.format(
table_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
table.update(
throughput={
'read': reads,
'write': writes
})
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < writes:
sns_message_types.append('scale-up')
message = []
if current_reads > reads:
message.append('{0} - Reads: DOWN from {1} to {2}\n'.format(
table_name, current_reads, reads))
elif current_reads < reads:
message.append('{0} - Reads: UP from {1} to {2}\n'.format(
table_name, current_reads, reads))
if current_writes > writes:
message.append('{0} - Writes: DOWN from {1} to {2}\n'.format(
table_name, current_writes, writes))
elif current_writes < writes:
message.append('{0} - Writes: UP from {1} to {2}\n'.format(
table_name, current_writes, writes))
sns.publish_table_notification(
key_name,
''.join(message),
sns_message_types,
subject='Updated provisioning for table {0}'.format(table_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException',
'ValidationException',
'ResourceInUseException']
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(
table_name, exception, error.body['message']))
else:
if 'message' in error.body:
msg = error.body['message']
else:
msg = error
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(table_name, exception, msg))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - Will retry to update provisioning '
'with only increases'.format(table_name))
update_table_provisioning(
table_name,
key_name,
reads,
writes,
retry_with_only_increase=True)
def update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Configuration option key name
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info("{0} - No need to scale up reads nor writes".format(table_name))
return
logger.info(
"{0} - Retrying to update provisioning, excluding any decreases. "
"Setting new reads to {1} and new writes to {2}".format(table_name, reads, writes)
)
# Check that we are in the right time frame
maintenance_windows = get_table_option(key_name, "maintenance_windows")
if maintenance_windows:
if not __is_table_maintenance_window(table_name, maintenance_windows):
logger.warning(
"{0} - We are outside a maintenace window. " "Will only perform up scaling activites".format(table_name)
)
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info("{0} - No need to scale up reads nor writes".format(table_name))
return
else:
logger.info("{0} - Current time is within maintenance window".format(table_name))
logger.info("{0} - Updating provisioning to {1} reads and {2} writes".format(table_name, reads, writes))
# Return if dry-run
if get_global_option("dry_run"):
return
try:
table.update(throughput={"read": reads, "write": writes})
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append("scale-down")
if current_reads < reads or current_writes < current_writes:
sns_message_types.append("scale-up")
message = ("{0} - Provisioning updated to {1} reads and {2} writes").format(table_name, reads, writes)
sns.publish_table_notification(
key_name, message, sns_message_types, subject="Updated provisioning for table {0}".format(table_name)
)
except JSONResponseError as error:
exception = error.body["__type"].split("#")[1]
know_exceptions = ["LimitExceededException", "ValidationException", "ResourceInUseException"]
if exception in know_exceptions:
logger.warning("{0} - {1}: {2}".format(table_name, exception, error.body["message"]))
else:
logger.error(
(
"{0} - Unhandled exception: {1}: {2}. "
"Please file a bug report at "
"https://github.com/sebdah/dynamic-dynamodb/issues"
).format(table_name, exception, error.body["message"])
)
if not retry_with_only_increase and exception == "LimitExceededException":
logger.info("{0} - Will retry to update provisioning " "with only increases".format(table_name))
update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=True)