def get_min_provisioned_reads(current_provisioning, table_name, key_name): """ Returns the minimum provisioned reads If the min_provisioned_reads value is less than current_provisioning * 2, then we return current_provisioning * 2, as DynamoDB cannot be scaled up with more than 100%. :type current_provisioning: int :param current_provisioning: The current provisioning :type table_name: str :param table_name: Name of the table :type key_name: str :param key_name: Name of the key :returns: int -- Minimum provisioned reads """ min_provisioned_reads = 1 if get_table_option(key_name, 'min_provisioned_reads'): min_provisioned_reads = int( get_table_option(key_name, 'min_provisioned_reads')) if min_provisioned_reads > int(current_provisioning * 2): min_provisioned_reads = int(current_provisioning * 2) logger.debug( '{0} - ' 'Cannot reach min_provisioned_reads as max scale up ' 'is 100% of current provisioning'.format( table_name)) logger.debug( '{0} - ' 'Setting min provisioned reads to {1}'.format( table_name, min_provisioned_reads)) return min_provisioned_reads
def decrease_reads_in_percent( current_provisioning, percent, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log :returns: int -- New provisioning value """ percent = float(percent) decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease min_provisioned_reads = __get_min_reads( current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, int(min_provisioned_reads))) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def decrease_writes_in_percent( current_provisioning, percent, key_name, table_name): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key :type table_name: str :param table_name: Name of the DynamoDB table """ decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease min_provisioned_writes = get_min_provisioned_writes( current_provisioning, table_name, key_name) if updated_provisioning < min_provisioned_writes: logger.info( '{0} - Reached provisioned writes min limit: {1:d}'.format( table_name, min_provisioned_writes)) return min_provisioned_writes logger.debug( '{0} - Write provisioning will be decreased to {1:d} units'.format( table_name, updated_provisioning)) return updated_provisioning
def get_consumed_read_units_percent(table_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( period=time_frame, start_time=datetime.utcnow()-timedelta(minutes=10, seconds=time_frame), end_time=datetime.utcnow()-timedelta(minutes=10), metric_name='ConsumedReadCapacityUnits', namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 consumed_read_units_percent = int(math.ceil( float(consumed_read_units) / \ float(get_provisioned_read_units(table_name)) * \ 100)) logger.info('{0} - Consumed read units: {1:d}%'.format( table_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_consumed_read_units(table_name, time_frame=300): """ Returns the number of consumed read units :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( time_frame, datetime.utcnow()-timedelta(minutes=10, seconds=time_frame), datetime.utcnow()-timedelta(minutes=10), 'ConsumedReadCapacityUnits', 'AWS/DynamoDB', ['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 logger.info('{0} - Consumed read units: {1:d}'.format( table_name, consumed_read_units)) return consumed_read_units
def run(self, check_interval=1): """ Run the daemon :type check_interval: int :param check_interval: Delay in seconds between checks """ while True: used_keys = set() table_names = set() configured_tables = configuration['tables'].keys() # Add regexp table names for table_name in core.dynamodb.list_tables(): for key_name in configured_tables: if re.match(key_name, table_name): logger.debug( "Table {0} match with config key {1}".format( table_name, key_name)) table_names.add((table_name, key_name)) used_keys.add(key_name) # Remove used tables for table_name in used_keys: configured_tables.remove(table_name) # Ensure provisioning for table_name, key_name in sorted(table_names): core.ensure_provisioning(table_name, key_name) # Sleep between the checks time.sleep(check_interval)
def decrease_reads_in_units( current_provisioning, units, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = __get_min_reads( current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, int(min_provisioned_reads))) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def decrease_writes_in_percent(current_provisioning, percent, min_provisioned_writes, log_tag): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ decrease = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning - decrease min_provisioned_writes = __get_min_writes(current_provisioning, min_provisioned_writes, log_tag) if updated_provisioning < min_provisioned_writes: logger.info('{0} - Reached provisioned writes min limit: {1:d}'.format( log_tag, min_provisioned_writes)) return min_provisioned_writes logger.debug( '{0} - Write provisioning will be decreased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def get_throttled_by_provisioned_read_event_percent(table_name, lookback_window_start=15): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric(table_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = float(metrics[0]['Sum']) / float(300) else: throttled_read_events = 0 try: throttled_by_provisioned_read_percent = ( float(throttled_read_events) / float(dynamodb.get_provisioned_table_read_units(table_name)) * 100) except JSONResponseError: raise logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format( table_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def get_throttled_by_consumed_write_percent(table_name, lookback_window_start=15): """ Returns the number of throttled write events in percent of consumption :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled write events by consumption """ try: metrics1 = __get_aws_metric( table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') metrics2 = __get_aws_metric( table_name, lookback_window_start, 'WriteThrottleEvents') except BotoServerError: raise if metrics1 and metrics2: throttled_by_consumed_write_percent = (((float(metrics2[0]['Sum'])/float(300)) / (float(metrics1[0]['Sum'])/float(300))) * 100) else: throttled_by_consumed_write_percent = 0 logger.info('{0} - Throttled write percent by consumption: {1:.2f}%'.format( table_name, throttled_by_consumed_write_percent)) return throttled_by_consumed_write_percent
def __get_min_writes(current_provisioning, min_provisioned_writes, log_tag): """ Get the minimum number of writes to current_provisioning :type current_provisioning: int :param current_provisioning: Current provisioned writes :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log :returns: int -- Minimum number of writes """ # Fallback value to ensure that we always have at least 1 read writes = 1 if min_provisioned_writes: writes = int(min_provisioned_writes) if writes > int(current_provisioning * 2): writes = int(current_provisioning * 2) logger.debug('{0} - ' 'Cannot reach min-provisioned-writes as max scale up ' 'is 100% of current provisioning'.format(log_tag)) logger.debug('{0} - Setting min provisioned writes to {1}'.format( log_tag, min_provisioned_writes)) return writes
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Decrease the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ decrease = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning - decrease logger.debug('Read provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_reads = get_min_provisioned_reads(table_name, current_provisioning, key_name) if min_provisioned_reads > 0: if updated_provisioning < min_provisioned_reads: logger.info('Reached provisioned reads min limit: {0:d}'.format( min_provisioned_reads)) return min_provisioned_reads return updated_provisioning
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Increase the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ increase = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning + increase logger.debug('Read provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_reads') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_reads')): logger.info('Reached provisioned reads max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_reads')))) return get_table_option(key_name, 'max_provisioned_reads') return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units, key_name): """ Increase the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) + int(units) logger.debug('Write provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_writes') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_writes')): logger.info('Reached provisioned writes max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_writes')))) return get_table_option(key_name, 'max_provisioned_writes') return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name): """ Decrease the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) - int(units) logger.debug('Write provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_writes = get_min_provisioned_writes( table_name, current_provisioning, key_name) if min_provisioned_writes > 0: if updated_provisioning < min_provisioned_writes: logger.info('Reached provisioned writes min limit: {0:d}'.format( min_provisioned_writes)) return min_provisioned_writes return updated_provisioning
def get_consumed_read_units(table_name, time_frame=300): """ Returns the number of consumed read units :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( time_frame, datetime.utcnow() - timedelta(minutes=10, seconds=time_frame), datetime.utcnow() - timedelta(minutes=10), 'ConsumedReadCapacityUnits', 'AWS/DynamoDB', ['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_read_units = 0 logger.info('{0} - Consumed read units: {1:d}'.format( table_name, consumed_read_units)) return consumed_read_units
def get_throttled_read_event_count( table_name, lookback_window_start=15, lookback_period=5): """ Returns the number of throttled read events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: int -- Number of throttled read events during the time period """ try: metrics = __get_aws_metric( table_name, lookback_window_start, lookback_period, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = int(metrics[0]['Sum']) else: throttled_read_events = 0 logger.info('{0} - Read throttle count: {1:d}'.format( table_name, throttled_read_events)) return throttled_read_events
def get_throttled_by_provisioned_read_event_percent(table_name, lookback_window_start=15): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric( table_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = float(metrics[0]['Sum'])/float(300) else: throttled_read_events = 0 try: throttled_by_provisioned_read_percent = (float(throttled_read_events) / float(dynamodb.get_provisioned_table_read_units(table_name)) * 100) except JSONResponseError: raise logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format( table_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def ensure_created(table_name, template_table_name): """ Ensure table has been created in DynamoDB based on given template table name :type table_name: str :param table_name: Name of the DynamoDB table :type template_table_name: str :param template_table_name: Name of the template DynamoDB table (that has hashkey, attribute definitions) """ try: desc = DYNAMODB_CONNECTION.describe_table(table_name)[u'Table'] except JSONResponseError: try : template_table = get_table( template_table_name ) template_table.describe() logger.info( '{0} - Create table with template table schema {1}, throughput {2}, indexes {3}, global_indexes {4}'.format( table_name, template_table.schema, template_table.throughput, template_table.indexes, template_table.global_indexes)) # Return if dry-run if get_global_option('dry_run'): return table = Table.create(table_name, schema=template_table.schema, throughput=template_table.throughput, indexes=template_table.indexes, global_indexes=template_table.global_indexes, connection=DYNAMODB_CONNECTION) except DynamoDBResponseError: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error( '{0} - Table {1} not found'.format(table_name, table_name)) raise
def get_throttled_by_consumed_write_percent(table_name, lookback_window_start=15): """ Returns the number of throttled write events in percent of consumption :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled write events by consumption """ try: metrics1 = __get_aws_metric(table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') metrics2 = __get_aws_metric(table_name, lookback_window_start, 'WriteThrottleEvents') except BotoServerError: raise if metrics1 and metrics2: throttled_by_consumed_write_percent = ( ((float(metrics2[0]['Sum']) / float(300)) / (float(metrics1[0]['Sum']) / float(300))) * 100) else: throttled_by_consumed_write_percent = 0 logger.info( '{0} - Throttled write percent by consumption: {1:.2f}%'.format( table_name, throttled_by_consumed_write_percent)) return throttled_by_consumed_write_percent
def decrease_reads_in_units(current_provisioning, units, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = __get_min_reads(current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info('{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, min_provisioned_reads)) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def __is_table_maintenance_window(table_name, maintenance_windows): """ Checks that the current time is within the maintenance window :type table_name: str :param table_name: Name of the DynamoDB table :type maintenance_windows: str :param maintenance_windows: Example: '00:00-01:00,10:00-11:00' :returns: bool -- True if within maintenance window """ # Example string '00:00-01:00,10:00-11:00' maintenance_window_list = [] for window in maintenance_windows.split(','): try: start, end = window.split('-', 1) except ValueError: logger.error( '{0} - Malformatted maintenance window'.format(table_name)) return False maintenance_window_list.append((start, end)) now = datetime.datetime.utcnow().strftime('%H%M') for maintenance_window in maintenance_window_list: start = ''.join(maintenance_window[0].split(':')) end = ''.join(maintenance_window[1].split(':')) if now >= start and now <= end: return True return False
def __is_maintenance_window(table_name, maintenance_windows): """ Checks that the current time is within the maintenance window :type table_name: str :param table_name: Name of the DynamoDB table :type maintenance_windows: str :param maintenance_windows: Example: '00:00-01:00,10:00-11:00' :returns: bool -- True if within maintenance window """ # Example string '00:00-01:00,10:00-11:00' maintenance_window_list = [] for window in maintenance_windows.split(','): try: start, end = window.split('-', 1) except ValueError: logger.error( '{0} - Malformatted maintenance window'.format(table_name)) return False maintenance_window_list.append((start, end)) now = datetime.datetime.utcnow().strftime('%H%M') for maintenance_window in maintenance_window_list: start = ''.join(maintenance_window[0].split(':')) end = ''.join(maintenance_window[1].split(':')) if now >= start and now <= end: return True return False
def get_throttled_read_event_count(table_name, gsi_name, time_frame=300): """ Returns the number of throttled read events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of throttled read events """ try: metrics = __get_aws_metric(table_name, gsi_name, time_frame, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = int(metrics[0]['Sum']) else: throttled_read_events = 0 logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format( table_name, gsi_name, throttled_read_events)) return throttled_read_events
def increase_writes_in_percent( current_provisioning, percent, max_provisioned_writes, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ increase = int(math.ceil(float(current_provisioning)*(float(percent)/100))) updated_provisioning = current_provisioning + increase if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ try: metrics = __get_aws_metric(table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits') except BotoServerError: raise if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_read_units = 0 try: consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float( dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name)) * 100)) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Decrease the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease logger.debug( 'Read provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_reads = get_min_provisioned_reads( table_name, current_provisioning, key_name) if min_provisioned_reads > 0: if updated_provisioning < min_provisioned_reads: logger.info('Reached provisioned reads min limit: {0:d}'.format( min_provisioned_reads)) return min_provisioned_reads return updated_provisioning
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Increase the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ increase = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning + increase logger.debug( 'Read provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_reads') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_reads')): logger.info('Reached provisioned reads max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_reads')))) return get_table_option(key_name, 'max_provisioned_reads') return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name): """ Decrease the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) - int(units) logger.debug( 'Write provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_writes = get_min_provisioned_writes( table_name, current_provisioning, key_name) if min_provisioned_writes > 0: if updated_provisioning < min_provisioned_writes: logger.info('Reached provisioned writes min limit: {0:d}'.format( min_provisioned_writes)) return min_provisioned_writes return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units, key_name): """ Increase the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) + int(units) logger.debug( 'Write provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_writes') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_writes')): logger.info('Reached provisioned writes max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_writes')))) return get_table_option(key_name, 'max_provisioned_writes') return updated_provisioning
def __get_aws_metric(table_name, lookback_window_start, metric_name): """ Returns a metric list from the AWS CloudWatch service, may return None if no metric exists :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: How many minutes to look at :type metric_name: str :param metric_name: Name of the metric to retrieve from CloudWatch :returns: list -- A list of time series data for the given metric, may be None if there was no data """ try: now = datetime.utcnow() start_time = now - timedelta(minutes=lookback_window_start) end_time = now - timedelta(minutes=lookback_window_start - 5) return cloudwatch_connection.get_metric_statistics( period=300, # Always look at 5 minutes windows start_time=start_time, end_time=end_time, metric_name=metric_name, namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') except BotoServerError as error: logger.error('Unknown boto error. Status: "{0}". ' 'Reason: "{1}". Message: {2}'.format( error.status, error.reason, error.message)) raise
def get_consumed_read_units_percent(table_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( period=time_frame, start_time=datetime.utcnow() - timedelta(minutes=10, seconds=time_frame), end_time=datetime.utcnow() - timedelta(minutes=10), metric_name='ConsumedReadCapacityUnits', namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_read_units = 0 consumed_read_units_percent = int(math.ceil( float(consumed_read_units) / \ float(get_provisioned_read_units(table_name)) * \ 100)) logger.info('{0} - Consumed read units: {1:d}%'.format( table_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_consumed_write_units_percent(table_name, lookback_window_start=15): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric( table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum'])/float(300))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float(dynamodb.get_provisioned_table_write_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed write units: {1:d}%'.format( table_name, consumed_write_units_percent)) return consumed_write_units_percent
def decrease_reads_in_units(current_provisioning, units, key_name, table_name): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key :type table_name: str :param table_name: Name of the DynamoDB table """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = get_min_provisioned_reads( current_provisioning, table_name, key_name) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( table_name, min_provisioned_reads)) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( table_name, updated_provisioning)) return updated_provisioning
def __get_min_writes(current_provisioning, min_provisioned_writes, log_tag): """ Get the minimum number of writes to current_provisioning :type current_provisioning: int :param current_provisioning: Current provisioned writes :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log :returns: int -- Minimum number of writes """ # Fallback value to ensure that we always have at least 1 read writes = 1 if min_provisioned_writes: writes = int(min_provisioned_writes) if writes > int(current_provisioning * 2): writes = int(current_provisioning * 2) logger.debug( '{0} - ' 'Cannot reach min-provisioned-writes as max scale up ' 'is 100% of current provisioning'.format(log_tag)) logger.debug( '{0} - Setting min provisioned writes to {1}'.format( log_tag, min_provisioned_writes)) return writes
def __ensure_provisioning_reads(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :returns: (bool, int) -- update_needed, updated_read_units """ update_needed = False updated_read_units = statistics.get_provisioned_read_units(table_name) consumed_read_units_percent = \ statistics.get_consumed_read_units_percent(table_name) if (consumed_read_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - Scaling down reads is not done when usage is at 0%'.format( table_name)) elif (consumed_read_units_percent >= get_table_option(key_name, 'reads_upper_threshold')): if get_table_option(key_name, 'increase_reads_unit') == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif (consumed_read_units_percent <= get_table_option(key_name, 'reads_lower_threshold')): if get_table_option(key_name, 'decrease_reads_unit') == 'percent': updated_provisioning = calculators.decrease_reads_in_percent( updated_read_units, get_table_option(key_name, 'decrease_reads_with'), key_name) else: updated_provisioning = calculators.decrease_reads_in_units( updated_read_units, get_table_option(key_name, 'decrease_reads_with'), key_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning return update_needed, int(updated_read_units)
def increase_writes_in_units( current_provisioning, units, max_provisioned_writes, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = __get_aws_metric( table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 try: consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float(dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name)) * 100)) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def __get_connection_cloudwatch(): """ Ensure connection to CloudWatch """ region = get_global_option('region') try: if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug('Authenticating to CloudWatch using ' 'credentials in configuration file') connection = cloudwatch.connect_to_region( region, aws_access_key_id=get_global_option('aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: logger.debug('Authenticating using boto\'s authentication handler') connection = cloudwatch.connect_to_region(region) except Exception as err: logger.error('Failed connecting to CloudWatch: {0}'.format(err)) logger.error('Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise logger.debug('Connected to CloudWatch in {0}'.format(region)) return connection
def get_consumed_write_units_percent(table_name, lookback_window_start=15): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric(table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum']) / float(300))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float(dynamodb.get_provisioned_table_write_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed write units: {1:d}%'.format( table_name, consumed_write_units_percent)) return consumed_write_units_percent
def get_consumed_write_units_percent(table_name, time_frame=300): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric( table_name, time_frame, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float(dynamodb.get_provisioned_table_write_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed write units: {1:d}%'.format( table_name, consumed_write_units_percent)) return consumed_write_units_percent
def get_throttled_read_event_count( table_name, gsi_name, lookback_window_start=15): """ Returns the number of throttled read events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: How many seconds to look at :returns: int -- Number of throttled read events """ try: metrics = __get_aws_metric( table_name, gsi_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = int(metrics[0]['Sum']) else: throttled_read_events = 0 logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format( table_name, gsi_name, throttled_read_events)) return throttled_read_events
def get_throttled_write_event_count(table_name, gsi_name, time_frame=300): """ Returns the number of throttled write events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of throttled write events """ try: metrics = __get_aws_metric( table_name, gsi_name, time_frame, 'WriteThrottleEvents') except BotoServerError: raise if metrics: throttled_write_events = int(metrics[0]['Sum']) else: throttled_write_events = 0 logger.info('{0} - GSI: {1} - Write throttle count: {2:d}'.format( table_name, gsi_name, throttled_write_events)) return throttled_write_events
def __get_aws_metric(table_name, time_frame, metric_name): """ Returns a metric list from the AWS CloudWatch service, may return None if no metric exists :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :type metric_name str :param metric_name Name of the metric to retrieve from CloudWatch :returns: list -- A list of time series data for the given metric, may be None if there was no data """ try: start_time = datetime.utcnow()-timedelta(minutes=10, seconds=time_frame) end_time = datetime.utcnow()-timedelta(minutes=10) return cloudwatch_connection.get_metric_statistics( period=time_frame, start_time=start_time, end_time=end_time, metric_name=metric_name, namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') except BotoServerError as error: logger.error( 'Unknown boto error. Status: "{0}". ' 'Reason: "{1}". Message: {2}'.format( error.status, error.reason, error.message)) raise
def __get_connection_cloudwatch(): """ Ensure connection to CloudWatch """ region = get_global_option('region') try: if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug( 'Authenticating to CloudWatch using ' 'credentials in configuration file') connection = cloudwatch.connect_to_region( region, aws_access_key_id=get_global_option('aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: logger.debug( 'Authenticating using boto\'s authentication handler') connection = cloudwatch.connect_to_region(region) except Exception as err: logger.error('Failed connecting to CloudWatch: {0}'.format(err)) logger.error( 'Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise logger.debug('Connected to CloudWatch in {0}'.format(region)) return connection
def __ensure_provisioning_writes(table_name): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :returns: (bool, int) -- update_needed, updated_write_units """ update_needed = False updated_write_units = statistics.get_provisioned_write_units(table_name) consumed_write_units_percent = \ statistics.get_consumed_write_units_percent(table_name) # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_table_option(table_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Scaling down writes is not done when usage is at 0%'.format( table_name)) elif (consumed_write_units_percent >= get_table_option(table_name, 'writes_upper_threshold')): if get_table_option(table_name, 'increase_writes_unit') == 'percent': updated_provisioning = calculators.increase_writes_in_percent( table_name, updated_write_units, get_table_option(table_name, 'increase_writes_with')) else: updated_provisioning = calculators.increase_writes_in_units( table_name, updated_write_units, get_table_option(table_name, 'increase_writes_with')) update_needed = True updated_write_units = updated_provisioning elif (consumed_write_units_percent <= get_table_option(table_name, 'writes_lower_threshold')): if get_table_option(table_name, 'decrease_writes_unit') == 'percent': updated_provisioning = calculators.decrease_writes_in_percent( table_name, updated_write_units, get_table_option(table_name, 'decrease_writes_with')) else: updated_provisioning = calculators.decrease_writes_in_units( table_name, updated_write_units, get_table_option(table_name, 'decrease_writes_with')) update_needed = True updated_write_units = updated_provisioning return update_needed, int(updated_write_units)
def run(self): """ Run the daemon :type check_interval: int :param check_interval: Delay in seconds between checks """ try: while True: execute() except Exception as error: logger.exception(error)
def execute_table_in_loop(table_name, table_key, table_logger): if not get_global_option('daemon') and get_global_option('run_once'): execute_table(table_name, table_key) else: t = threading.currentThread() while getattr(t, "do_run", True): execute_table(table_name, table_key, table_logger) logger.debug('Sleeping {0} seconds until next check'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval'))
def get_provisioned_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ table = dynamodb.get_table(table_name) logger.debug('{0} - Provisioned write units: {1:d}'.format( table_name, table.write_units)) return int(table.write_units)
def execute(): """ Ensure provisioning """ boto_server_error_retries = 3 # Ensure provisioning for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()): logger.debug("Table: " + table_name) execute_in_thread(table_name, table_key) # Sleep between the checks if not get_global_option('run_once'): logger.debug('Sleeping {0} seconds until next thread check'.format(60)) time.sleep(60)
def __get_connection_dynamodb(retries=3): """ Ensure connection to DynamoDB :type retries: int :param retries: Number of times to retry to connect to DynamoDB """ connected = False while not connected: try: if (configuration['global']['aws_access_key_id'] and configuration['global']['aws_secret_access_key']): connection = dynamodb.connect_to_region( configuration['global']['region'], aws_access_key_id=configuration['global']['aws_access_key_id'], aws_secret_access_key=\ configuration['global']['aws_secret_access_key']) else: connection = dynamodb.connect_to_region( configuration['global']['region']) connected = True except Exception as err: logger.error('Failed to connect to DynamoDB: {0}'.format(err)) if retries == 0: logger.error( 'Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise else: logger.error('Retrying in 5 seconds') retries -= 1 time.sleep(5) logger.debug('Connected to DynamoDB') return connection
def get_tables_and_gsis(): """ Get a set of tables and gsis and their configuration keys :returns: set -- A set of tuples (table_name, table_conf_key) """ table_names = set() configured_tables = get_configured_tables() not_used_tables = set(configured_tables) # Add regexp table names for table_instance in list_tables(): for key_name in configured_tables: try: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format( table_instance.table_name, key_name)) table_names.add((table_instance.table_name, key_name)) not_used_tables.discard(key_name) else: logger.debug( "Table {0} did not match with config key {1}".format( table_instance.table_name, key_name)) except re.error: logger.error( 'Invalid regular expression: "{0}"'.format(key_name)) sys.exit(1) if not_used_tables: logger.warning('No tables matching the following configured ' 'tables found: {0}'.format(', '.join(not_used_tables))) return sorted(table_names)
def list_tables(): """ Return list of DynamoDB tables available from AWS :returns: list -- List of DynamoDB tables """ tables = [] try: table_list = DYNAMODB_CONNECTION.list_tables() while True: for table_name in table_list[u'TableNames']: tables.append(get_table(table_name)) if u'LastEvaluatedTableName' in table_list: table_list = DYNAMODB_CONNECTION.list_tables( table_list[u'LastEvaluatedTableName']) else: break except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error('No tables found') elif dynamodb_error == 'AccessDeniedException': logger.debug( 'Your AWS API keys lack access to listing tables. ' 'That is an issue if you are trying to use regular ' 'expressions in your table configuration.') elif dynamodb_error == 'UnrecognizedClientException': logger.error( 'Invalid security token. Are your AWS API keys correct?') else: logger.error( ( 'Unhandled exception: {0}: {1}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( dynamodb_error, error.body['message'])) except JSONResponseError as error: logger.error('Communication error: {0}'.format(error)) sys.exit(1) return tables
def increase_writes_in_percent(current_provisioning, percent, max_provisioned_writes, consumed_write_units_percent, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type consumed_write_units_percent: float :param consumed_write_units_percent: Number of consumed write units :type log_tag: str :param log_tag: Prefix for the log """ current_provisioning = float(current_provisioning) consumed_write_units_percent = float(consumed_write_units_percent) percent = float(percent) consumption_based_current_provisioning = \ int(math.ceil(current_provisioning*(consumed_write_units_percent/100))) if consumption_based_current_provisioning > current_provisioning: increase = int( math.ceil(consumption_based_current_provisioning * (percent / 100))) updated_provisioning = consumption_based_current_provisioning + increase else: increase = int(math.ceil(current_provisioning * (float(percent) / 100))) updated_provisioning = current_provisioning + increase if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def get_provisioned_table_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ try: desc = DYNAMODB_CONNECTION.describe_table(table_name) except JSONResponseError: raise write_units = int( desc[u'Table'][u'ProvisionedThroughput'][u'WriteCapacityUnits']) logger.debug('{0} - Currently provisioned write units: {1:d}'.format( table_name, write_units)) return write_units
def get_table(table_name): """ Return the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :returns: boto.dynamodb.table.Table """ try: table = Table(table_name, connection=DYNAMODB_CONNECTION) except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error('{0} - Table {1} not found'.format( table_name, table_name)) raise return table
def __publish(topic, message, subject=None): """ Publish a message to a SNS topic :type topic: str :param topic: SNS topic to publish the message to :type message: str :param message: Message to send via SNS :type subject: str :param subject: Subject to use for e-mail notifications :returns: None """ try: SNS_CONNECTION.publish(topic=topic, message=message, subject=subject) logger.info('Sent SNS notification to {0}'.format(topic)) except BotoServerError as error: logger.error('Problem sending SNS notification: {0}'.format( error.message)) return
def __calculate_always_decrease_rw_values(table_name, gsi_name, read_units, provisioned_reads, write_units, provisioned_writes): """ Calculate values for always-decrease-rw-together This will only return reads and writes decreases if both reads and writes are lower than the current provisioning :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type read_units: int :param read_units: New read unit provisioning :type provisioned_reads: int :param provisioned_reads: Currently provisioned reads :type write_units: int :param write_units: New write unit provisioning :type provisioned_writes: int :param provisioned_writes: Currently provisioned writes :returns: (int, int) -- (reads, writes) """ if read_units < provisioned_reads and write_units < provisioned_writes: return (read_units, write_units) if read_units < provisioned_reads: logger.info( '{0} - GSI: {1} - Reads could be decreased, ' 'but we are waiting for writes to get lower than the threshold ' 'before scaling down'.format(table_name, gsi_name)) read_units = provisioned_reads elif write_units < provisioned_writes: logger.info( '{0} - GSI: {1} - Writes could be decreased, ' 'but we are waiting for reads to get lower than the threshold ' 'before scaling down'.format(table_name, gsi_name)) write_units = provisioned_writes return (read_units, write_units)