def decrease_writes_in_percent(current_provisioning, percent, min_provisioned_writes, log_tag): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ decrease = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning - decrease min_provisioned_writes = __get_min_writes(current_provisioning, min_provisioned_writes, log_tag) if updated_provisioning < min_provisioned_writes: logger.info('{0} - Reached provisioned writes min limit: {1:d}'.format( log_tag, min_provisioned_writes)) return min_provisioned_writes logger.debug( '{0} - Write provisioning will be decreased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def decrease_writes_in_percent( current_provisioning, percent, key_name, table_name): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key :type table_name: str :param table_name: Name of the DynamoDB table """ decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease min_provisioned_writes = get_min_provisioned_writes( current_provisioning, table_name, key_name) if updated_provisioning < min_provisioned_writes: logger.info( '{0} - Reached provisioned writes min limit: {1:d}'.format( table_name, min_provisioned_writes)) return min_provisioned_writes logger.debug( '{0} - Write provisioning will be decreased to {1:d} units'.format( table_name, updated_provisioning)) return updated_provisioning
def get_throttled_read_event_count( table_name, lookback_window_start=15, lookback_period=5): """ Returns the number of throttled read events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: int -- Number of throttled read events during the time period """ try: metrics = __get_aws_metric( table_name, lookback_window_start, lookback_period, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = int(metrics[0]['Sum']) else: throttled_read_events = 0 logger.info('{0} - Read throttle count: {1:d}'.format( table_name, throttled_read_events)) return throttled_read_events
def update_table_provisioning(table_name, reads, writes): """""" table = get_table(table_name) try: table.update( throughput={ 'read': reads, 'write': writes }) logger.info( '{0} - Provisioning updated to {1} reads and {2} writes'.format( table_name, reads, writes)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = [ 'LimitExceededException', 'ValidationException', 'ResourceInUseException'] if exception in know_exceptions: logger.warning('{0} - {1}: {2}'.format( table_name, exception, error.body['message'])) else: logger.error( ( '{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format(table_name, exception, error.body['message']))
def get_consumed_read_units_percent(table_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( period=time_frame, start_time=datetime.utcnow()-timedelta(minutes=10, seconds=time_frame), end_time=datetime.utcnow()-timedelta(minutes=10), metric_name='ConsumedReadCapacityUnits', namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 consumed_read_units_percent = int(math.ceil( float(consumed_read_units) / \ float(get_provisioned_read_units(table_name)) * \ 100)) logger.info('{0} - Consumed read units: {1:d}%'.format( table_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_consumed_read_units(table_name, time_frame=300): """ Returns the number of consumed read units :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( time_frame, datetime.utcnow()-timedelta(minutes=10, seconds=time_frame), datetime.utcnow()-timedelta(minutes=10), 'ConsumedReadCapacityUnits', 'AWS/DynamoDB', ['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 logger.info('{0} - Consumed read units: {1:d}'.format( table_name, consumed_read_units)) return consumed_read_units
def get_throttled_by_provisioned_read_event_percent(table_name, lookback_window_start=15): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric( table_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = float(metrics[0]['Sum'])/float(300) else: throttled_read_events = 0 try: throttled_by_provisioned_read_percent = (float(throttled_read_events) / float(dynamodb.get_provisioned_table_read_units(table_name)) * 100) except JSONResponseError: raise logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format( table_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def get_throttled_by_provisioned_read_event_percent(table_name, lookback_window_start=15): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric(table_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = float(metrics[0]['Sum']) / float(300) else: throttled_read_events = 0 try: throttled_by_provisioned_read_percent = ( float(throttled_read_events) / float(dynamodb.get_provisioned_table_read_units(table_name)) * 100) except JSONResponseError: raise logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format( table_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Increase the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ increase = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning + increase logger.debug('Read provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_reads') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_reads')): logger.info('Reached provisioned reads max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_reads')))) return get_table_option(key_name, 'max_provisioned_reads') return updated_provisioning
def get_throttled_by_consumed_write_percent(table_name, lookback_window_start=15): """ Returns the number of throttled write events in percent of consumption :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled write events by consumption """ try: metrics1 = __get_aws_metric( table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') metrics2 = __get_aws_metric( table_name, lookback_window_start, 'WriteThrottleEvents') except BotoServerError: raise if metrics1 and metrics2: throttled_by_consumed_write_percent = (((float(metrics2[0]['Sum'])/float(300)) / (float(metrics1[0]['Sum'])/float(300))) * 100) else: throttled_by_consumed_write_percent = 0 logger.info('{0} - Throttled write percent by consumption: {1:.2f}%'.format( table_name, throttled_by_consumed_write_percent)) return throttled_by_consumed_write_percent
def increase_writes_in_units(table_name, current_provisioning, units, key_name): """ Increase the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) + int(units) logger.debug('Write provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_writes') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_writes')): logger.info('Reached provisioned writes max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_writes')))) return get_table_option(key_name, 'max_provisioned_writes') return updated_provisioning
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Decrease the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ decrease = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning - decrease logger.debug('Read provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_reads = get_min_provisioned_reads(table_name, current_provisioning, key_name) if min_provisioned_reads > 0: if updated_provisioning < min_provisioned_reads: logger.info('Reached provisioned reads min limit: {0:d}'.format( min_provisioned_reads)) return min_provisioned_reads return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name): """ Decrease the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) - int(units) logger.debug('Write provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_writes = get_min_provisioned_writes( table_name, current_provisioning, key_name) if min_provisioned_writes > 0: if updated_provisioning < min_provisioned_writes: logger.info('Reached provisioned writes min limit: {0:d}'.format( min_provisioned_writes)) return min_provisioned_writes return updated_provisioning
def get_consumed_read_units(table_name, time_frame=300): """ Returns the number of consumed read units :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( time_frame, datetime.utcnow() - timedelta(minutes=10, seconds=time_frame), datetime.utcnow() - timedelta(minutes=10), 'ConsumedReadCapacityUnits', 'AWS/DynamoDB', ['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_read_units = 0 logger.info('{0} - Consumed read units: {1:d}'.format( table_name, consumed_read_units)) return consumed_read_units
def get_consumed_read_units_percent(table_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( period=time_frame, start_time=datetime.utcnow() - timedelta(minutes=10, seconds=time_frame), end_time=datetime.utcnow() - timedelta(minutes=10), metric_name='ConsumedReadCapacityUnits', namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_read_units = 0 consumed_read_units_percent = int(math.ceil( float(consumed_read_units) / \ float(get_provisioned_read_units(table_name)) * \ 100)) logger.info('{0} - Consumed read units: {1:d}%'.format( table_name, consumed_read_units_percent)) return consumed_read_units_percent
def ensure_created(table_name, template_table_name): """ Ensure table has been created in DynamoDB based on given template table name :type table_name: str :param table_name: Name of the DynamoDB table :type template_table_name: str :param template_table_name: Name of the template DynamoDB table (that has hashkey, attribute definitions) """ try: desc = DYNAMODB_CONNECTION.describe_table(table_name)[u'Table'] except JSONResponseError: try : template_table = get_table( template_table_name ) template_table.describe() logger.info( '{0} - Create table with template table schema {1}, throughput {2}, indexes {3}, global_indexes {4}'.format( table_name, template_table.schema, template_table.throughput, template_table.indexes, template_table.global_indexes)) # Return if dry-run if get_global_option('dry_run'): return table = Table.create(table_name, schema=template_table.schema, throughput=template_table.throughput, indexes=template_table.indexes, global_indexes=template_table.global_indexes, connection=DYNAMODB_CONNECTION) except DynamoDBResponseError: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error( '{0} - Table {1} not found'.format(table_name, table_name)) raise
def get_consumed_write_units_percent(table_name, lookback_window_start=15): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric( table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum'])/float(300))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float(dynamodb.get_provisioned_table_write_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed write units: {1:d}%'.format( table_name, consumed_write_units_percent)) return consumed_write_units_percent
def get_throttled_read_event_count(table_name, gsi_name, lookback_window_start=15): """ Returns the number of throttled read events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: How many seconds to look at :returns: int -- Number of throttled read events """ try: metrics = __get_aws_metric(table_name, gsi_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = int(metrics[0]['Sum']) else: throttled_read_events = 0 logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format( table_name, gsi_name, throttled_read_events)) return throttled_read_events
def increase_writes_in_percent( current_provisioning, percent, max_provisioned_writes, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ increase = int(math.ceil(float(current_provisioning)*(float(percent)/100))) updated_provisioning = current_provisioning + increase if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def increase_writes_in_units( current_provisioning, units, max_provisioned_writes, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def ensure_provisioning(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name """ if get_global_option('circuit_breaker_url'): if __circuit_breaker_is_open(): logger.warning('Circuit breaker is OPEN!') return None read_update_needed, updated_read_units = __ensure_provisioning_reads( table_name, key_name) write_update_needed, updated_write_units = __ensure_provisioning_writes( table_name, key_name) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) update_throughput(table_name, updated_read_units, updated_write_units, key_name) else: logger.info('{0} - No need to change provisioning'.format(table_name))
def get_consumed_write_units_percent(table_name, gsi_name, time_frame=300): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric(table_name, gsi_name, time_frame, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum']) / float(time_frame))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float( dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name)) * 100)) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed write units: {2:d}%'.format( table_name, gsi_name, consumed_write_units_percent)) return consumed_write_units_percent
def get_consumed_write_units_percent(table_name, lookback_window_start=15): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric(table_name, lookback_window_start, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum']) / float(300))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float(dynamodb.get_provisioned_table_write_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed write units: {1:d}%'.format( table_name, consumed_write_units_percent)) return consumed_write_units_percent
def increase_writes_in_percent(current_provisioning, percent, max_provisioned_writes, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ increase = int( math.ceil(float(current_provisioning) * (float(percent) / 100))) updated_provisioning = current_provisioning + increase if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option('daemon'): daemon = DynamicDynamoDBDaemon( '{0}/dynamic-dynamodb.{1}.pid'.format( get_global_option('pid_file_dir'), get_global_option('instance'))) if get_global_option('daemon') == 'start': daemon.start() elif get_global_option('daemon') == 'stop': logger.debug('Stopping daemon') daemon.stop() logger.info('Daemon stopped') sys.exit(0) elif get_global_option('daemon') == 'restart': daemon.restart() elif get_global_option('daemon') in ['foreground', 'fg']: daemon.run() else: print('Valid options for --daemon are start, stop and restart') sys.exit(1) else: while True: execute() except Exception as error: logger.exception(error)
def decrease_reads_in_units( current_provisioning, units, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = __get_min_reads( current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, int(min_provisioned_reads))) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def decrease_reads_in_percent( current_provisioning, percent, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log :returns: int -- New provisioning value """ percent = float(percent) decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease min_provisioned_reads = __get_min_reads( current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, int(min_provisioned_reads))) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def decrease_reads_in_units(current_provisioning, units, key_name, table_name): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key :type table_name: str :param table_name: Name of the DynamoDB table """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = get_min_provisioned_reads( current_provisioning, table_name, key_name) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( table_name, min_provisioned_reads)) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( table_name, updated_provisioning)) return updated_provisioning
def get_throttled_write_event_count(table_name, gsi_name, time_frame=300): """ Returns the number of throttled write events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of throttled write events """ try: metrics = __get_aws_metric(table_name, gsi_name, time_frame, 'WriteThrottleEvents') except BotoServerError: raise if metrics: throttled_write_events = int(metrics[0]['Sum']) else: throttled_write_events = 0 logger.info('{0} - GSI: {1} - Write throttle count: {2:d}'.format( table_name, gsi_name, throttled_write_events)) return throttled_write_events
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Increase the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ increase = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning + increase logger.debug( 'Read provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_reads') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_reads')): logger.info('Reached provisioned reads max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_reads')))) return get_table_option(key_name, 'max_provisioned_reads') return updated_provisioning
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Decrease the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease logger.debug( 'Read provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_reads = get_min_provisioned_reads( table_name, current_provisioning, key_name) if min_provisioned_reads > 0: if updated_provisioning < min_provisioned_reads: logger.info('Reached provisioned reads min limit: {0:d}'.format( min_provisioned_reads)) return min_provisioned_reads return updated_provisioning
def increase_writes_in_units(current_provisioning, units, max_provisioned_writes, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units, key_name): """ Increase the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) + int(units) logger.debug( 'Write provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_writes') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_writes')): logger.info('Reached provisioned writes max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_writes')))) return get_table_option(key_name, 'max_provisioned_writes') return updated_provisioning
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = __get_aws_metric( table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 try: consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float(dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name)) * 100)) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format( table_name, gsi_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_throttled_by_consumed_write_percent(table_name, gsi_name, lookback_window_start=15): """ Returns the number of throttled write events in percent of consumption :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :returns: float -- Percent of throttled write events by consumption """ try: metrics1 = __get_aws_metric(table_name, gsi_name, lookback_window_start, 'ConsumedWriteCapacityUnits') metrics2 = __get_aws_metric(table_name, gsi_name, lookback_window_start, 'WriteThrottleEvents') except BotoServerError: raise if metrics1 and metrics2: throttled_by_consumed_write_percent = ( ((float(metrics2[0]['Sum']) / float(300)) / (float(metrics1[0]['Sum']) / float(300))) * 100) else: throttled_by_consumed_write_percent = 0 logger.info('{0} - GSI: {1} - Throttled write percent ' 'by consumption: {2:.2f}%'.format( table_name, gsi_name, throttled_by_consumed_write_percent)) return throttled_by_consumed_write_percent
def get_throttled_read_event_count( table_name, gsi_name, lookback_window_start=15): """ Returns the number of throttled read events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: How many seconds to look at :returns: int -- Number of throttled read events """ try: metrics = __get_aws_metric( table_name, gsi_name, lookback_window_start, 'ReadThrottleEvents') except BotoServerError: raise if metrics: throttled_read_events = int(metrics[0]['Sum']) else: throttled_read_events = 0 logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format( table_name, gsi_name, throttled_read_events)) return throttled_read_events
def __ensure_provisioning_reads(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :returns: (bool, int) -- update_needed, updated_read_units """ update_needed = False updated_read_units = statistics.get_provisioned_read_units(table_name) consumed_read_units_percent = \ statistics.get_consumed_read_units_percent(table_name) if (consumed_read_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - Scaling down reads is not done when usage is at 0%'.format( table_name)) elif (consumed_read_units_percent >= get_table_option(key_name, 'reads_upper_threshold')): if get_table_option(key_name, 'increase_reads_unit') == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif (consumed_read_units_percent <= get_table_option(key_name, 'reads_lower_threshold')): if get_table_option(key_name, 'decrease_reads_unit') == 'percent': updated_provisioning = calculators.decrease_reads_in_percent( updated_read_units, get_table_option(key_name, 'decrease_reads_with'), key_name) else: updated_provisioning = calculators.decrease_reads_in_units( updated_read_units, get_table_option(key_name, 'decrease_reads_with'), key_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning return update_needed, int(updated_read_units)
def get_consumed_write_units_percent(table_name, time_frame=300): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed writes """ try: metrics = __get_aws_metric( table_name, time_frame, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: consumed_write_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_write_units = 0 try: consumed_write_units_percent = int( math.ceil( float(consumed_write_units) / float(dynamodb.get_provisioned_table_write_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed write units: {1:d}%'.format( table_name, consumed_write_units_percent)) return consumed_write_units_percent
def ensure_provisioning(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name """ if get_global_option('circuit_breaker_url'): if circuit_breaker.is_open(): logger.warning('Circuit breaker is OPEN!') return None read_update_needed, updated_read_units = __ensure_provisioning_reads( table_name, key_name) write_update_needed, updated_write_units = __ensure_provisioning_writes( table_name, key_name) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput( table_name, updated_read_units, updated_write_units, key_name) else: logger.info('{0} - No need to change provisioning'.format(table_name))
def get_throttled_write_event_count(table_name, gsi_name, time_frame=300): """ Returns the number of throttled write events during a given time frame :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of throttled write events """ try: metrics = __get_aws_metric( table_name, gsi_name, time_frame, 'WriteThrottleEvents') except BotoServerError: raise if metrics: throttled_write_events = int(metrics[0]['Sum']) else: throttled_write_events = 0 logger.info('{0} - GSI: {1} - Write throttle count: {2:d}'.format( table_name, gsi_name, throttled_write_events)) return throttled_write_events
def decrease_reads_in_units(current_provisioning, units, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = __get_min_reads(current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info('{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, min_provisioned_reads)) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name): """ Decrease the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) - int(units) logger.debug( 'Write provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_writes = get_min_provisioned_writes( table_name, current_provisioning, key_name) if min_provisioned_writes > 0: if updated_provisioning < min_provisioned_writes: logger.info('Reached provisioned writes min limit: {0:d}'.format( min_provisioned_writes)) return min_provisioned_writes return updated_provisioning
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option("daemon"): daemon = DynamicDynamoDBDaemon( "{0}/dynamic-dynamodb.{1}.pid".format(get_global_option("pid_file_dir"), get_global_option("instance")) ) if get_global_option("daemon") == "start": daemon.start() elif get_global_option("daemon") == "stop": logger.debug("Stopping daemon") daemon.stop() logger.info("Daemon stopped") sys.exit(0) elif get_global_option("daemon") == "restart": daemon.restart() elif get_global_option("daemon") in ["foreground", "fg"]: daemon.run() else: print("Valid options for --daemon are start, stop and restart") sys.exit(1) else: while True: execute() except Exception as error: logger.exception(error)
def ensure_provisioning(table_name, key_name, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option('circuit_breaker_url') or get_table_option( key_name, 'circuit_breaker_url'): if circuit_breaker.is_open(table_name, key_name): logger.warning('Circuit breaker is OPEN!') return (0, 0) # Handle throughput alarm checks __ensure_provisioning_alarm(table_name, key_name) try: read_update_needed, updated_read_units, num_consec_read_checks = \ __ensure_provisioning_reads( table_name, key_name, num_consec_read_checks) write_update_needed, updated_write_units, num_consec_write_checks = \ __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks) if read_update_needed: num_consec_read_checks = 0 if write_update_needed: num_consec_write_checks = 0 # Handle throughput updates if read_update_needed or write_update_needed: logger.info('{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput(table_name, key_name, updated_read_units, updated_write_units) else: logger.info( '{0} - No need to change provisioning'.format(table_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option('show_config'): print json.dumps(config.get_configuration(), indent=2) elif get_global_option('daemon'): daemon = DynamicDynamoDBDaemon( '{0}/dynamic-dynamodb.{1}.pid'.format( get_global_option('pid_file_dir'), get_global_option('instance'))) if get_global_option('daemon') == 'start': logger.debug('Starting daemon') try: daemon.start() logger.info('Daemon started') except IOError as error: logger.error( 'Could not create pid file: {0}'.format(error)) logger.error('Daemon not started') elif get_global_option('daemon') == 'stop': logger.debug('Stopping daemon') daemon.stop() logger.info('Daemon stopped') sys.exit(0) elif get_global_option('daemon') == 'restart': logger.debug('Restarting daemon') daemon.restart() logger.info('Daemon restarted') elif get_global_option('daemon') in ['foreground', 'fg']: logger.debug('Starting daemon in foreground') daemon.run() logger.info('Daemon started in foreground') else: print( 'Valid options for --daemon are start, ' 'stop, restart, and foreground') sys.exit(1) else: if get_global_option('run_once'): execute() else: while True: execute() except Exception as error: logger.exception(error) except KeyboardInterrupt: while threading.active_count() > 1: for thread in table_threads.values(): thread.do_run = False logger.info('Waiting for all threads... {}s'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval')) raise
def __ensure_provisioning_writes(table_name): """ Ensure that provisioning of writes is correct :type table_name: str :param table_name: Name of the DynamoDB table :returns: (bool, int) -- update_needed, updated_write_units """ update_needed = False updated_write_units = statistics.get_provisioned_write_units(table_name) consumed_write_units_percent = \ statistics.get_consumed_write_units_percent(table_name) # Check if we should update write provisioning if (consumed_write_units_percent == 0 and not get_table_option(table_name, 'allow_scaling_down_writes_on_0_percent')): logger.info( '{0} - Scaling down writes is not done when usage is at 0%'.format( table_name)) elif (consumed_write_units_percent >= get_table_option(table_name, 'writes_upper_threshold')): if get_table_option(table_name, 'increase_writes_unit') == 'percent': updated_provisioning = calculators.increase_writes_in_percent( table_name, updated_write_units, get_table_option(table_name, 'increase_writes_with')) else: updated_provisioning = calculators.increase_writes_in_units( table_name, updated_write_units, get_table_option(table_name, 'increase_writes_with')) update_needed = True updated_write_units = updated_provisioning elif (consumed_write_units_percent <= get_table_option(table_name, 'writes_lower_threshold')): if get_table_option(table_name, 'decrease_writes_unit') == 'percent': updated_provisioning = calculators.decrease_writes_in_percent( table_name, updated_write_units, get_table_option(table_name, 'decrease_writes_with')) else: updated_provisioning = calculators.decrease_writes_in_units( table_name, updated_write_units, get_table_option(table_name, 'decrease_writes_with')) update_needed = True updated_write_units = updated_provisioning return update_needed, int(updated_write_units)
def ensure_provisioning( table_name, key_name, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option('circuit_breaker_url'): if circuit_breaker.is_open(): logger.warning('Circuit breaker is OPEN!') return (0, 0) try: read_update_needed, updated_read_units, num_consec_read_checks = \ __ensure_provisioning_reads( table_name, key_name, num_consec_read_checks) write_update_needed, updated_write_units, num_consec_write_checks = \ __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput( table_name, key_name, updated_read_units, updated_write_units) else: logger.info('{0} - No need to change provisioning'.format( table_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug('{0} - GSI: {1} - GSI status is {2}'.format( table_name, gsi_name, gsi_status)) if gsi_status != 'ACTIVE': logger.warning( '{0} - GSI: {1} - Not performing throughput changes when GSI ' 'status is {2}'.format(table_name, gsi_name, gsi_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - GSI: {1} - No changes to perform'.format( table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def increase_reads_in_units( current_provisioning, units, table_name, table_key, gsi_name, gsi_key): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Name of the key :returns: int -- New provisioning value """ updated_provisioning = 0 if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'): max_provisioned_reads = int(get_gsi_option( table_key, gsi_key, 'max_provisioned_reads')) else: max_provisioned_reads = 0 if (max_provisioned_reads > 0 and updated_provisioning > max_provisioned_reads): logger.info( '{0} - GSI: {1} - ' 'Reached provisioned reads max limit: {2:d}'.format( table_name, gsi_name, max_provisioned_reads)) return max_provisioned_reads logger.debug( '{0} - GSI: {1} - ' 'Read provisioning will be increased to {2:d} units'.format( table_name, gsi_name, updated_provisioning)) return updated_provisioning
def __update_throughput(table_name, key_name, read_units, write_units): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_table_read_units(table_name) current_wu = dynamodb.get_provisioned_table_write_units(table_name) except JSONResponseError: raise # Check table status try: table_status = dynamodb.get_table_status(table_name) except JSONResponseError: raise logger.debug('{0} - Table status is {1}'.format(table_name, table_status)) if table_status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is {1}'.format(table_name, table_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - No changes to perform'.format(table_name)) return dynamodb.update_table_provisioning( table_name, key_name, int(read_units), int(write_units))
def execute_in_thread(table_name, table_key): if table_name in table_threads: thread = table_threads.get(table_name) if thread.isAlive(): logger.debug("Thread {} still running".format(table_name)) else: logger.error("Thread {} was stopped!".format(table_name)) sys.exit(1) else: table_logger = log_handler.getTableLogger(table_name) logger.info("Start new thread: " + table_name) thread = threading.Thread(target=execute_table_in_loop, args=[table_name, table_key, table_logger]) table_threads[table_name] = thread thread.start()
def increase_reads_in_percent(current_provisioning, percent, max_provisioned_reads, consumed_read_units_percent, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :type max_provisioned_reads: int :param max_provisioned_reads: Configured max provisioned reads :returns: int -- New provisioning value :type consumed_read_units_percent: float :param consumed_read_units_percent: Number of consumed read units :type log_tag: str :param log_tag: Prefix for the log """ current_provisioning = float(current_provisioning) consumed_read_units_percent = float(consumed_read_units_percent) percent = float(percent) consumption_based_current_provisioning = \ float(math.ceil(current_provisioning*(consumed_read_units_percent/100))) if consumption_based_current_provisioning > current_provisioning: increase = int( math.ceil(consumption_based_current_provisioning * (percent / 100))) updated_provisioning = consumption_based_current_provisioning + increase else: increase = int(math.ceil(current_provisioning * (percent / 100))) updated_provisioning = current_provisioning + increase if max_provisioned_reads > 0: if updated_provisioning > max_provisioned_reads: logger.info( '{0} - Reached provisioned reads max limit: {1}'.format( log_tag, max_provisioned_reads)) return max_provisioned_reads logger.debug( '{0} - Read provisioning will be increased to {1} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def __calculate_always_decrease_rw_values(table_name, gsi_name, read_units, provisioned_reads, write_units, provisioned_writes): """ Calculate values for always-decrease-rw-together This will only return reads and writes decreases if both reads and writes are lower than the current provisioning :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type read_units: int :param read_units: New read unit provisioning :type provisioned_reads: int :param provisioned_reads: Currently provisioned reads :type write_units: int :param write_units: New write unit provisioning :type provisioned_writes: int :param provisioned_writes: Currently provisioned writes :returns: (int, int) -- (reads, writes) """ if read_units < provisioned_reads and write_units < provisioned_writes: return (read_units, write_units) if read_units < provisioned_reads: logger.info( '{0} - GSI: {1} - Reads could be decreased, ' 'but we are waiting for writes to get lower than the threshold ' 'before scaling down'.format(table_name, gsi_name)) read_units = provisioned_reads elif write_units < provisioned_writes: logger.info( '{0} - GSI: {1} - Writes could be decreased, ' 'but we are waiting for reads to get lower than the threshold ' 'before scaling down'.format(table_name, gsi_name)) write_units = provisioned_writes return (read_units, write_units)
def get_throttled_by_provisioned_read_event_percent(table_name, gsi_name, lookback_window_start=15, lookback_period=5): """ Returns the number of throttled read events in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: float -- Percent of throttled read events by provisioning """ try: metrics = __get_aws_metric(table_name, gsi_name, lookback_window_start, lookback_period, 'ReadThrottleEvents') except BotoServerError: raise if metrics: lookback_seconds = lookback_period * 60 throttled_read_events = (float(metrics[0]['Sum']) / float(lookback_seconds)) else: throttled_read_events = 0 try: gsi_read_units = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) throttled_by_provisioned_read_percent = (float(throttled_read_events) / float(gsi_read_units) * 100) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Throttled read percent ' 'by provision: {2:.2f}%'.format( table_name, gsi_name, throttled_by_provisioned_read_percent)) return throttled_by_provisioned_read_percent
def __publish(topic, message, subject=None): """ Publish a message to a SNS topic :type topic: str :param topic: SNS topic to publish the message to :type message: str :param message: Message to send via SNS :type subject: str :param subject: Subject to use for e-mail notifications :returns: None """ try: SNS_CONNECTION.publish(topic=topic, message=message, subject=subject) logger.info('Sent SNS notification to {0}'.format(topic)) except BotoServerError as error: logger.error('Problem sending SNS notification: {0}'.format( error.message)) return
def get_consumed_write_units_percent(table_name, gsi_name, lookback_window_start=15, lookback_period=5): """ Returns the number of consumed write units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :type lookback_window_start: int :param lookback_window_start: Relative start time for the CloudWatch metric :type lookback_period: int :param lookback_period: Number of minutes to look at :returns: float -- Number of consumed writes as a percentage of provisioned writes """ try: metrics = __get_aws_metric(table_name, gsi_name, lookback_window_start, lookback_period, 'ConsumedWriteCapacityUnits') except BotoServerError: raise if metrics: lookback_seconds = lookback_period * 60 consumed_write_units = (float(metrics[0]['Sum']) / float(lookback_seconds)) else: consumed_write_units = 0 try: gsi_write_units = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) consumed_write_units_percent = (float(consumed_write_units) / float(gsi_write_units) * 100) except JSONResponseError: raise logger.info('{0} - GSI: {1} - Consumed write units: {2:.2f}%'.format( table_name, gsi_name, consumed_write_units_percent)) return consumed_write_units_percent
def increase_writes_in_units(current_provisioning, units, max_provisioned_writes, consumed_write_units_percent, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type consumed_write_units_percent: float :param consumed_write_units_percent: Number of consumed write units :type log_tag: str :param log_tag: Prefix for the log """ units = int(units) current_provisioning = float(current_provisioning) consumed_write_units_percent = float(consumed_write_units_percent) consumption_based_current_provisioning = \ int(math.ceil(current_provisioning*(consumed_write_units_percent/100))) if consumption_based_current_provisioning > current_provisioning: updated_provisioning = consumption_based_current_provisioning + units else: updated_provisioning = int(current_provisioning) + units if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option('show_config'): print json.dumps(config.get_configuration(), indent=2) elif get_global_option('daemon'): daemon = DynamicDynamoDBDaemon( '{0}/dynamic-dynamodb.{1}.pid'.format( get_global_option('pid_file_dir'), get_global_option('instance'))) if get_global_option('daemon') == 'start': logger.debug('Starting daemon') try: daemon.start() logger.info('Daemon started') except IOError as error: logger.error( 'Could not create pid file: {0}'.format(error)) logger.error('Daemon not started') elif get_global_option('daemon') == 'stop': logger.debug('Stopping daemon') daemon.stop() logger.info('Daemon stopped') sys.exit(0) elif get_global_option('daemon') == 'restart': logger.debug('Restarting daemon') daemon.restart() logger.info('Daemon restarted') elif get_global_option('daemon') in ['foreground', 'fg']: logger.debug('Starting daemon in foreground') daemon.run() logger.info('Daemon started in foreground') else: print( 'Valid options for --daemon are start, ' 'stop, restart, and foreground') sys.exit(1) else: if get_global_option('run_once'): execute() else: while True: execute() except Exception as error: logger.exception(error)