def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key): """ Ensure that provisioning alarm threshold is not exceeded :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name """ lookback_window_start = get_gsi_option( table_key, gsi_key, 'lookback_window_start') consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent( table_name, gsi_name, lookback_window_start) consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent( table_name, gsi_name, lookback_window_start) reads_upper_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold') reads_lower_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold') writes_upper_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold') writes_lower_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold') # Check upper alarm thresholds upper_alert_triggered = False upper_alert_message = [] if (reads_upper_alarm_threshold > 0 and consumed_read_units_percent >= reads_upper_alarm_threshold): upper_alert_triggered = True upper_alert_message.append( '{0} - GSI: {1} - Consumed Read Capacity {2:d}% ' 'was greater than or equal to the upper alarm ' 'threshold {3:d}%\n'.format( table_name, gsi_name, consumed_read_units_percent, reads_upper_alarm_threshold)) if (writes_upper_alarm_threshold > 0 and consumed_write_units_percent >= writes_upper_alarm_threshold): upper_alert_triggered = True upper_alert_message.append( '{0} - GSI: {1} - Consumed Write Capacity {2:d}% ' 'was greater than or equal to the upper alarm ' 'threshold {3:d}%\n'.format( table_name, gsi_name, consumed_write_units_percent, writes_upper_alarm_threshold)) # Check lower alarm thresholds lower_alert_triggered = False lower_alert_message = [] if (reads_lower_alarm_threshold > 0 and consumed_read_units_percent < reads_lower_alarm_threshold): lower_alert_triggered = True lower_alert_message.append( '{0} - GSI: {1} - Consumed Read Capacity {2:d}% ' 'was below the lower alarm threshold {3:d}%\n'.format( table_name, gsi_name, consumed_read_units_percent, reads_lower_alarm_threshold)) if (writes_lower_alarm_threshold > 0 and consumed_write_units_percent < writes_lower_alarm_threshold): lower_alert_triggered = True lower_alert_message.append( '{0} - GSI: {1} - Consumed Write Capacity {2:d}% ' 'was below the lower alarm threshold {3:d}%\n'.format( table_name, gsi_name, consumed_write_units_percent, writes_lower_alarm_threshold)) # Send alert if needed if upper_alert_triggered: logger.info( '{0} - GSI: {1} - Will send high provisioning alert'.format( table_name, gsi_name)) sns.publish_gsi_notification( table_key, gsi_key, ''.join(upper_alert_message), ['high-throughput-alarm'], subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format( table_name, gsi_name)) elif lower_alert_triggered: logger.info( '{0} - GSI: {1} - Will send low provisioning alert'.format( table_name, gsi_name)) sns.publish_gsi_notification( table_key, gsi_key, ''.join(lower_alert_message), ['low-throughput-alarm'], subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format( table_name, gsi_name)) else: logger.debug( '{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format( table_name, gsi_name))
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) # Check that we are in the right time frame m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows') if m_windows: if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows): logger.warning( '{0} - GSI: {1} - We are outside a maintenace window. ' 'Will only perform up scaling activites'.format( table_name, gsi_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info('{0} - GSI: {1} - ' 'No need to scale up reads nor writes'.format( table_name, gsi_name)) return else: logger.info('{0} - GSI: {1} - ' 'Current time is within maintenance window'.format( table_name, gsi_name)) logger.info('{0} - GSI: {1} - ' 'Updating provisioning to {2} reads and {3} writes'.format( table_name, gsi_name, reads, writes)) # Return if dry-run if get_global_option('dry_run'): return try: DYNAMODB_CONNECTION.update_table(table_name=table_name, global_secondary_index_updates=[{ "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } }]) message = ('{0} - GSI: {1} - Provisioning updated to ' '{2} reads and {3} writes').format(table_name, gsi_name, reads, writes) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < current_writes: sns_message_types.append('scale-up') sns.publish_gsi_notification( table_key, gsi_key, message, sns_message_types, subject='Updated provisioning for GSI {0}'.format(gsi_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ('{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues').format( table_name, gsi_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info('{0} - GSI: {1} - Will retry to update provisioning ' 'with only increases'.format(table_name, gsi_name)) update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True)
def update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name)) current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name)) # Make sure we aren't scaling down if we turned off downscaling if (not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling') or not get_gsi_option( table_key, gsi_key, 'enable_writes_down_scaling')): if (not get_gsi_option( table_key, gsi_key, 'enable_reads_down_scaling') and current_reads > reads): reads = current_reads if (not get_gsi_option( table_key, gsi_key, 'enable_writes_down_scaling') and current_writes > writes): writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info( '{0} - No need to scale up reads nor writes'.format( table_name)) return if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info( '{0} - GSI: {1} - No need to scale up reads nor writes'.format( table_name, gsi_name)) return logger.info( '{0} - GSI: {1} - Retrying to update provisioning, ' 'excluding any decreases. ' 'Setting new reads to {2} and new writes to {3}'.format( table_name, gsi_name, reads, writes)) # Check that we are in the right time frame m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows') if m_windows: if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows): logger.warning( '{0} - GSI: {1} - We are outside a maintenace window. ' 'Will only perform up scaling activites'.format( table_name, gsi_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info( '{0} - GSI: {1} - ' 'No need to scale up reads nor writes'.format( table_name, gsi_name)) return else: logger.info( '{0} - GSI: {1} - ' 'Current time is within maintenance window'.format( table_name, gsi_name)) logger.info( '{0} - GSI: {1} - ' 'Updating provisioning to {2} reads and {3} writes'.format( table_name, gsi_name, reads, writes)) # Return if dry-run if get_global_option('dry_run'): return try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } } ]) message = [] if current_reads > reads: message.append( '{0} - GSI: {1} - Reads: DOWN from {2} to {3}\n'.format( table_name, gsi_name, current_reads, reads)) elif current_reads < reads: message.append( '{0} - GSI: {1} - Reads: UP from {2} to {3}\n'.format( table_name, gsi_name, current_reads, reads)) if current_writes > writes: message.append( '{0} - GSI: {1} - Writes: DOWN from {2} to {3}\n'.format( table_name, gsi_name, current_writes, writes)) elif current_writes < writes: message.append( '{0} - GSI: {1} - Writes: UP from {2} to {3}\n'.format( table_name, gsi_name, current_writes, writes)) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < writes: sns_message_types.append('scale-up') sns.publish_gsi_notification( table_key, gsi_key, ''.join(message), sns_message_types, subject='Updated provisioning for GSI {0}'.format(gsi_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ( '{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, gsi_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info( '{0} - GSI: {1} - Will retry to update provisioning ' 'with only increases'.format(table_name, gsi_name)) update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True)
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key): """ Ensure that provisioning alarm threshold is not exceeded :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name """ lookback_window_start = get_gsi_option(table_key, gsi_key, 'lookback_window_start') consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent( table_name, gsi_name, lookback_window_start) consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent( table_name, gsi_name, lookback_window_start) reads_upper_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'reads-upper-alarm-threshold') reads_lower_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'reads-lower-alarm-threshold') writes_upper_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'writes-upper-alarm-threshold') writes_lower_alarm_threshold = \ get_gsi_option(table_key, gsi_key, 'writes-lower-alarm-threshold') # Check upper alarm thresholds upper_alert_triggered = False upper_alert_message = [] if (reads_upper_alarm_threshold > 0 and consumed_read_units_percent >= reads_upper_alarm_threshold): upper_alert_triggered = True upper_alert_message.append( '{0} - GSI: {1} - Consumed Read Capacity {2:d}% ' 'was greater than or equal to the upper alarm ' 'threshold {3:d}%\n'.format(table_name, gsi_name, consumed_read_units_percent, reads_upper_alarm_threshold)) if (writes_upper_alarm_threshold > 0 and consumed_write_units_percent >= writes_upper_alarm_threshold): upper_alert_triggered = True upper_alert_message.append( '{0} - GSI: {1} - Consumed Write Capacity {2:d}% ' 'was greater than or equal to the upper alarm ' 'threshold {3:d}%\n'.format(table_name, gsi_name, consumed_write_units_percent, writes_upper_alarm_threshold)) # Check lower alarm thresholds lower_alert_triggered = False lower_alert_message = [] if (reads_lower_alarm_threshold > 0 and consumed_read_units_percent < reads_lower_alarm_threshold): lower_alert_triggered = True lower_alert_message.append( '{0} - GSI: {1} - Consumed Read Capacity {2:d}% ' 'was below the lower alarm threshold {3:d}%\n'.format( table_name, gsi_name, consumed_read_units_percent, reads_lower_alarm_threshold)) if (writes_lower_alarm_threshold > 0 and consumed_write_units_percent < writes_lower_alarm_threshold): lower_alert_triggered = True lower_alert_message.append( '{0} - GSI: {1} - Consumed Write Capacity {2:d}% ' 'was below the lower alarm threshold {3:d}%\n'.format( table_name, gsi_name, consumed_write_units_percent, writes_lower_alarm_threshold)) # Send alert if needed if upper_alert_triggered: logger.info( '{0} - GSI: {1} - Will send high provisioning alert'.format( table_name, gsi_name)) sns.publish_gsi_notification( table_key, gsi_key, ''.join(upper_alert_message), ['high-throughput-alarm'], subject='ALARM: High Throughput for Table {0} - GSI: {1}'.format( table_name, gsi_name)) elif lower_alert_triggered: logger.info('{0} - GSI: {1} - Will send low provisioning alert'.format( table_name, gsi_name)) sns.publish_gsi_notification( table_key, gsi_key, ''.join(lower_alert_message), ['low-throughput-alarm'], subject='ALARM: Low Throughput for Table {0} - GSI: {1}'.format( table_name, gsi_name)) else: logger.debug( '{0} - GSI: {1} - Throughput alarm thresholds not crossed'.format( table_name, gsi_name))
def __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key): """ Ensure that provisioning alarm threshold is not exceeded :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name """ consumed_read_units_percent = gsi_stats.get_consumed_read_units_percent(table_name, gsi_name) consumed_write_units_percent = gsi_stats.get_consumed_write_units_percent(table_name, gsi_name) reads_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-upper-alarm-threshold") reads_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "reads-lower-alarm-threshold") writes_upper_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-upper-alarm-threshold") writes_lower_alarm_threshold = get_gsi_option(table_key, gsi_key, "writes-lower-alarm-threshold") # Check upper alarm thresholds upper_alert_triggered = False upper_alert_message = [] if reads_upper_alarm_threshold > 0 and consumed_read_units_percent >= reads_upper_alarm_threshold: upper_alert_triggered = True upper_alert_message.append( "{0} - GSI: {1} - Consumed Read Capacity {2:d}% " "was greater than or equal to the upper alarm " "threshold {3:d}%\n".format(table_name, gsi_name, consumed_read_units_percent, reads_upper_alarm_threshold) ) if writes_upper_alarm_threshold > 0 and consumed_write_units_percent >= writes_upper_alarm_threshold: upper_alert_triggered = True upper_alert_message.append( "{0} - GSI: {1} - Consumed Write Capacity {2:d}% " "was greater than or equal to the upper alarm " "threshold {3:d}%\n".format( table_name, gsi_name, consumed_write_units_percent, writes_upper_alarm_threshold ) ) # Check lower alarm thresholds lower_alert_triggered = False lower_alert_message = [] if reads_lower_alarm_threshold > 0 and consumed_read_units_percent < reads_lower_alarm_threshold: lower_alert_triggered = True lower_alert_message.append( "{0} - GSI: {1} - Consumed Read Capacity {2:d}% " "was below the lower alarm threshold {3:d}%\n".format( table_name, gsi_name, consumed_read_units_percent, reads_lower_alarm_threshold ) ) if writes_lower_alarm_threshold > 0 and consumed_write_units_percent < writes_lower_alarm_threshold: lower_alert_triggered = True lower_alert_message.append( "{0} - GSI: {1} - Consumed Write Capacity {2:d}% " "was below the lower alarm threshold {3:d}%\n".format( table_name, gsi_name, consumed_write_units_percent, writes_lower_alarm_threshold ) ) # Send alert if needed if upper_alert_triggered: logger.info("{0} - GSI: {1} - Will send high provisioning alert".format(table_name, gsi_name)) sns.publish_gsi_notification( table_key, gsi_key, "".join(upper_alert_message), ["high-throughput-alarm"], subject="ALARM: High Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name), ) elif lower_alert_triggered: logger.info("{0} - GSI: {1} - Will send low provisioning alert".format(table_name, gsi_name)) sns.publish_gsi_notification( table_key, gsi_key, "".join(lower_alert_message), ["low-throughput-alarm"], subject="ALARM: Low Throughput for Table {0} - GSI: {1}".format(table_name, gsi_name), ) else: logger.debug("{0} - GSI: {1} - Throughput alarm thresholds not crossed".format(table_name, gsi_name))
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name)) current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info("{0} - GSI: {1} - No need to scale up reads nor writes".format(table_name, gsi_name)) return logger.info( "{0} - GSI: {1} - Retrying to update provisioning, excluding any decreases. " "Setting new reads to {2} and new writes to {3}".format(table_name, gsi_name, reads, writes) ) # Check that we are in the right time frame m_windows = get_gsi_option(table_key, gsi_key, "maintenance_windows") if m_windows: if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows): logger.warning( "{0} - GSI: {1} - We are outside a maintenace window. " "Will only perform up scaling activites".format(table_name, gsi_name) ) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info("{0} - GSI: {1} - " "No need to scale up reads nor writes".format(table_name, gsi_name)) return else: logger.info("{0} - GSI: {1} - " "Current time is within maintenance window".format(table_name, gsi_name)) logger.info( "{0} - GSI: {1} - " "Updating provisioning to {2} reads and {3} writes".format(table_name, gsi_name, reads, writes) ) # Return if dry-run if get_global_option("dry_run"): return try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": {"ReadCapacityUnits": reads, "WriteCapacityUnits": writes}, } } ], ) message = ("{0} - GSI: {1} - Provisioning updated to " "{2} reads and {3} writes").format( table_name, gsi_name, reads, writes ) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append("scale-down") if current_reads < reads or current_writes < current_writes: sns_message_types.append("scale-up") sns.publish_gsi_notification( table_key, gsi_key, message, sns_message_types, subject="Updated provisioning for GSI {0}".format(gsi_name) ) except JSONResponseError as error: exception = error.body["__type"].split("#")[1] know_exceptions = ["LimitExceededException"] if exception in know_exceptions: logger.warning("{0} - GSI: {1} - {2}: {3}".format(table_name, gsi_name, exception, error.body["message"])) else: logger.error( ( "{0} - GSI: {1} - Unhandled exception: {2}: {3}. " "Please file a bug report at " "https://github.com/sebdah/dynamic-dynamodb/issues" ).format(table_name, gsi_name, exception, error.body["message"]) ) if not retry_with_only_increase and exception == "LimitExceededException": logger.info( "{0} - GSI: {1} - Will retry to update provisioning " "with only increases".format(table_name, gsi_name) ) update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True )