def decrease_writes_in_percent(current_provisioning, percent,
min_provisioned_writes, log_tag):
""" Decrease the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type min_provisioned_writes: int
:param min_provisioned_writes: Configured min provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
decrease = int(float(current_provisioning) * (float(percent) / 100))
updated_provisioning = current_provisioning - decrease
min_provisioned_writes = __get_min_writes(current_provisioning,
min_provisioned_writes, log_tag)
if updated_provisioning < min_provisioned_writes:
logger.info('{0} - Reached provisioned writes min limit: {1:d}'.format(
log_tag, min_provisioned_writes))
return min_provisioned_writes
logger.debug(
'{0} - Write provisioning will be decreased to {1:d} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def decrease_writes_in_percent(
current_provisioning, percent, key_name, table_name):
""" Decrease the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
:type table_name: str
:param table_name: Name of the DynamoDB table
"""
decrease = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning - decrease
min_provisioned_writes = get_min_provisioned_writes(
current_provisioning,
table_name,
key_name)
if updated_provisioning < min_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes min limit: {1:d}'.format(
table_name,
min_provisioned_writes))
return min_provisioned_writes
logger.debug(
'{0} - Write provisioning will be decreased to {1:d} units'.format(
table_name,
updated_provisioning))
return updated_provisioning
def get_throttled_read_event_count(
table_name, lookback_window_start=15, lookback_period=5):
""" Returns the number of throttled read events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: int -- Number of throttled read events during the time period
"""
try:
metrics = __get_aws_metric(
table_name,
lookback_window_start,
lookback_period,
'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = int(metrics[0]['Sum'])
else:
throttled_read_events = 0
logger.info('{0} - Read throttle count: {1:d}'.format(
table_name, throttled_read_events))
return throttled_read_events
def update_table_provisioning(table_name, reads, writes):
""""""
table = get_table(table_name)
try:
table.update(
throughput={
'read': reads,
'write': writes
})
logger.info(
'{0} - Provisioning updated to {1} reads and {2} writes'.format(
table_name, reads, writes))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException',
'ValidationException',
'ResourceInUseException']
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(
table_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(table_name, exception, error.body['message']))
def get_consumed_read_units_percent(table_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
period=time_frame,
start_time=datetime.utcnow()-timedelta(minutes=10, seconds=time_frame),
end_time=datetime.utcnow()-timedelta(minutes=10),
metric_name='ConsumedReadCapacityUnits',
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_read_units = 0
consumed_read_units_percent = int(math.ceil(
float(consumed_read_units) / \
float(get_provisioned_read_units(table_name)) * \
100))
logger.info('{0} - Consumed read units: {1:d}%'.format(
table_name, consumed_read_units_percent))
return consumed_read_units_percent
def get_consumed_read_units(table_name, time_frame=300):
""" Returns the number of consumed read units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
time_frame,
datetime.utcnow()-timedelta(minutes=10, seconds=time_frame),
datetime.utcnow()-timedelta(minutes=10),
'ConsumedReadCapacityUnits',
'AWS/DynamoDB',
['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_read_units = 0
logger.info('{0} - Consumed read units: {1:d}'.format(
table_name, consumed_read_units))
return consumed_read_units
def get_throttled_by_provisioned_read_event_percent(table_name, lookback_window_start=15):
""" Returns the number of throttled read events in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled read events by provisioning
"""
try:
metrics = __get_aws_metric(
table_name, lookback_window_start, 'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = float(metrics[0]['Sum'])/float(300)
else:
throttled_read_events = 0
try: throttled_by_provisioned_read_percent = (float(throttled_read_events) /
float(dynamodb.get_provisioned_table_read_units(table_name)) *
100)
except JSONResponseError:
raise
logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format(
table_name, throttled_by_provisioned_read_percent))
return throttled_by_provisioned_read_percent
def get_throttled_by_provisioned_read_event_percent(table_name,
lookback_window_start=15):
""" Returns the number of throttled read events in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled read events by provisioning
"""
try:
metrics = __get_aws_metric(table_name, lookback_window_start,
'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = float(metrics[0]['Sum']) / float(300)
else:
throttled_read_events = 0
try:
throttled_by_provisioned_read_percent = (
float(throttled_read_events) /
float(dynamodb.get_provisioned_table_read_units(table_name)) * 100)
except JSONResponseError:
raise
logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format(
table_name, throttled_by_provisioned_read_percent))
return throttled_by_provisioned_read_percent
def increase_reads_in_percent(table_name, current_provisioning, percent,
key_name):
""" Increase the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
increase = int(float(current_provisioning) * (float(percent) / 100))
updated_provisioning = current_provisioning + increase
logger.debug('Read provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_reads') > 0:
if (updated_provisioning > get_table_option(key_name,
'max_provisioned_reads')):
logger.info('Reached provisioned reads max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_reads'))))
return get_table_option(key_name, 'max_provisioned_reads')
return updated_provisioning
def get_throttled_by_consumed_write_percent(table_name, lookback_window_start=15):
""" Returns the number of throttled write events in percent of consumption
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled write events by consumption
"""
try:
metrics1 = __get_aws_metric(
table_name, lookback_window_start, 'ConsumedWriteCapacityUnits')
metrics2 = __get_aws_metric(
table_name, lookback_window_start, 'WriteThrottleEvents')
except BotoServerError:
raise
if metrics1 and metrics2:
throttled_by_consumed_write_percent = (((float(metrics2[0]['Sum'])/float(300)) /
(float(metrics1[0]['Sum'])/float(300))) * 100)
else:
throttled_by_consumed_write_percent = 0
logger.info('{0} - Throttled write percent by consumption: {1:.2f}%'.format(
table_name, throttled_by_consumed_write_percent))
return throttled_by_consumed_write_percent
def increase_writes_in_units(table_name, current_provisioning, units,
key_name):
""" Increase the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) + int(units)
logger.debug('Write provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_writes') > 0:
if (updated_provisioning > get_table_option(key_name,
'max_provisioned_writes')):
logger.info('Reached provisioned writes max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_writes'))))
return get_table_option(key_name, 'max_provisioned_writes')
return updated_provisioning
def decrease_reads_in_percent(table_name, current_provisioning, percent,
key_name):
""" Decrease the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
decrease = int(float(current_provisioning) * (float(percent) / 100))
updated_provisioning = current_provisioning - decrease
logger.debug('Read provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_reads = get_min_provisioned_reads(table_name,
current_provisioning,
key_name)
if min_provisioned_reads > 0:
if updated_provisioning < min_provisioned_reads:
logger.info('Reached provisioned reads min limit: {0:d}'.format(
min_provisioned_reads))
return min_provisioned_reads
return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units,
key_name):
""" Decrease the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) - int(units)
logger.debug('Write provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_writes = get_min_provisioned_writes(
table_name, current_provisioning, key_name)
if min_provisioned_writes > 0:
if updated_provisioning < min_provisioned_writes:
logger.info('Reached provisioned writes min limit: {0:d}'.format(
min_provisioned_writes))
return min_provisioned_writes
return updated_provisioning
def get_consumed_read_units(table_name, time_frame=300):
""" Returns the number of consumed read units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
time_frame,
datetime.utcnow() - timedelta(minutes=10, seconds=time_frame),
datetime.utcnow() - timedelta(minutes=10),
'ConsumedReadCapacityUnits',
'AWS/DynamoDB', ['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_read_units = 0
logger.info('{0} - Consumed read units: {1:d}'.format(
table_name, consumed_read_units))
return consumed_read_units
def get_consumed_read_units_percent(table_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
period=time_frame,
start_time=datetime.utcnow() -
timedelta(minutes=10, seconds=time_frame),
end_time=datetime.utcnow() - timedelta(minutes=10),
metric_name='ConsumedReadCapacityUnits',
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_read_units = 0
consumed_read_units_percent = int(math.ceil(
float(consumed_read_units) / \
float(get_provisioned_read_units(table_name)) * \
100))
logger.info('{0} - Consumed read units: {1:d}%'.format(
table_name, consumed_read_units_percent))
return consumed_read_units_percent
def ensure_created(table_name, template_table_name):
""" Ensure table has been created in DynamoDB based on given template table name
:type table_name: str
:param table_name: Name of the DynamoDB table
:type template_table_name: str
:param template_table_name: Name of the template DynamoDB table (that has hashkey, attribute definitions)
"""
try:
desc = DYNAMODB_CONNECTION.describe_table(table_name)[u'Table']
except JSONResponseError:
try :
template_table = get_table( template_table_name )
template_table.describe()
logger.info(
'{0} - Create table with template table schema {1}, throughput {2}, indexes {3}, global_indexes {4}'.format(
table_name, template_table.schema, template_table.throughput, template_table.indexes, template_table.global_indexes))
# Return if dry-run
if get_global_option('dry_run'):
return
table = Table.create(table_name, schema=template_table.schema,
throughput=template_table.throughput, indexes=template_table.indexes,
global_indexes=template_table.global_indexes,
connection=DYNAMODB_CONNECTION)
except DynamoDBResponseError:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'ResourceNotFoundException':
logger.error(
'{0} - Table {1} not found'.format(table_name, table_name))
raise
def get_consumed_write_units_percent(table_name, lookback_window_start=15):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(
table_name, lookback_window_start, 'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum'])/float(300)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_table_write_units(table_name)) *
100))
except JSONResponseError:
raise
logger.info('{0} - Consumed write units: {1:d}%'.format(
table_name, consumed_write_units_percent))
return consumed_write_units_percent
def get_throttled_read_event_count(table_name,
gsi_name,
lookback_window_start=15):
""" Returns the number of throttled read events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: How many seconds to look at
:returns: int -- Number of throttled read events
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, lookback_window_start,
'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = int(metrics[0]['Sum'])
else:
throttled_read_events = 0
logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format(
table_name, gsi_name, throttled_read_events))
return throttled_read_events
def increase_writes_in_percent(
current_provisioning, percent, max_provisioned_writes, log_tag):
""" Increase the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
increase = int(math.ceil(float(current_provisioning)*(float(percent)/100)))
updated_provisioning = current_provisioning + increase
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag,
max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag,
updated_provisioning))
return updated_provisioning
def increase_writes_in_units(
current_provisioning, units, max_provisioned_writes, log_tag):
""" Increase the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
if int(units) > int(current_provisioning):
updated_provisioning = 2 * int(current_provisioning)
else:
updated_provisioning = int(current_provisioning) + int(units)
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag,
max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag,
updated_provisioning))
return updated_provisioning
def ensure_provisioning(table_name, key_name):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
"""
if get_global_option('circuit_breaker_url'):
if __circuit_breaker_is_open():
logger.warning('Circuit breaker is OPEN!')
return None
read_update_needed, updated_read_units = __ensure_provisioning_reads(
table_name, key_name)
write_update_needed, updated_write_units = __ensure_provisioning_writes(
table_name, key_name)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
update_throughput(table_name, updated_read_units, updated_write_units, key_name)
else:
logger.info('{0} - No need to change provisioning'.format(table_name))
def get_consumed_write_units_percent(table_name, gsi_name, time_frame=300):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, time_frame,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) / float(
dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)) * 100))
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed write units: {2:d}%'.format(
table_name, gsi_name, consumed_write_units_percent))
return consumed_write_units_percent
def get_consumed_write_units_percent(table_name, lookback_window_start=15):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(table_name, lookback_window_start,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum']) / float(300)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_table_write_units(table_name)) *
100))
except JSONResponseError:
raise
logger.info('{0} - Consumed write units: {1:d}%'.format(
table_name, consumed_write_units_percent))
return consumed_write_units_percent
def increase_writes_in_percent(current_provisioning, percent,
max_provisioned_writes, log_tag):
""" Increase the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
increase = int(
math.ceil(float(current_provisioning) * (float(percent) / 100)))
updated_provisioning = current_provisioning + increase
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag, max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def main():
""" Main function called from dynamic-dynamodb """
try:
if get_global_option('daemon'):
daemon = DynamicDynamoDBDaemon(
'{0}/dynamic-dynamodb.{1}.pid'.format(
get_global_option('pid_file_dir'),
get_global_option('instance')))
if get_global_option('daemon') == 'start':
daemon.start()
elif get_global_option('daemon') == 'stop':
logger.debug('Stopping daemon')
daemon.stop()
logger.info('Daemon stopped')
sys.exit(0)
elif get_global_option('daemon') == 'restart':
daemon.restart()
elif get_global_option('daemon') in ['foreground', 'fg']:
daemon.run()
else:
print('Valid options for --daemon are start, stop and restart')
sys.exit(1)
else:
while True:
execute()
except Exception as error:
logger.exception(error)
def decrease_reads_in_units(
current_provisioning, units, min_provisioned_reads, log_tag):
""" Decrease the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type min_provisioned_reads: int
:param min_provisioned_reads: Configured min provisioned reads
:type log_tag: str
:param log_tag: Prefix for the log
"""
updated_provisioning = int(current_provisioning) - int(units)
min_provisioned_reads = __get_min_reads(
current_provisioning,
min_provisioned_reads,
log_tag)
if updated_provisioning < min_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads min limit: {1:d}'.format(
log_tag,
int(min_provisioned_reads)))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
log_tag,
int(updated_provisioning)))
return updated_provisioning
def decrease_reads_in_percent(
current_provisioning, percent, min_provisioned_reads, log_tag):
""" Decrease the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:type min_provisioned_reads: int
:param min_provisioned_reads: Configured min provisioned reads
:type log_tag: str
:param log_tag: Prefix for the log
:returns: int -- New provisioning value
"""
percent = float(percent)
decrease = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning - decrease
min_provisioned_reads = __get_min_reads(
current_provisioning,
min_provisioned_reads,
log_tag)
if updated_provisioning < min_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads min limit: {1:d}'.format(
log_tag, int(min_provisioned_reads)))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
log_tag, int(updated_provisioning)))
return updated_provisioning
def decrease_reads_in_units(current_provisioning, units, key_name, table_name):
""" Decrease the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
:type table_name: str
:param table_name: Name of the DynamoDB table
"""
updated_provisioning = int(current_provisioning) - int(units)
min_provisioned_reads = get_min_provisioned_reads(
current_provisioning,
table_name,
key_name)
if updated_provisioning < min_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads min limit: {1:d}'.format(
table_name,
min_provisioned_reads))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
table_name,
updated_provisioning))
return updated_provisioning
def get_throttled_write_event_count(table_name, gsi_name, time_frame=300):
""" Returns the number of throttled write events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of throttled write events
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, time_frame,
'WriteThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_write_events = int(metrics[0]['Sum'])
else:
throttled_write_events = 0
logger.info('{0} - GSI: {1} - Write throttle count: {2:d}'.format(
table_name, gsi_name, throttled_write_events))
return throttled_write_events
def main():
""" Main function called from dynamic-dynamodb """
try:
if get_global_option('daemon'):
daemon = DynamicDynamoDBDaemon(
'{0}/dynamic-dynamodb.{1}.pid'.format(
get_global_option('pid_file_dir'),
get_global_option('instance')))
if get_global_option('daemon') == 'start':
daemon.start()
elif get_global_option('daemon') == 'stop':
logger.debug('Stopping daemon')
daemon.stop()
logger.info('Daemon stopped')
sys.exit(0)
elif get_global_option('daemon') == 'restart':
daemon.restart()
elif get_global_option('daemon') in ['foreground', 'fg']:
daemon.run()
else:
print('Valid options for --daemon are start, stop and restart')
sys.exit(1)
else:
while True:
execute()
except Exception as error:
logger.exception(error)
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name):
""" Increase the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
increase = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning + increase
logger.debug(
'Read provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_reads') > 0:
if (updated_provisioning >
get_table_option(key_name, 'max_provisioned_reads')):
logger.info('Reached provisioned reads max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_reads'))))
return get_table_option(key_name, 'max_provisioned_reads')
return updated_provisioning
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name):
""" Decrease the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
decrease = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning - decrease
logger.debug(
'Read provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_reads = get_min_provisioned_reads(
table_name,
current_provisioning,
key_name)
if min_provisioned_reads > 0:
if updated_provisioning < min_provisioned_reads:
logger.info('Reached provisioned reads min limit: {0:d}'.format(
min_provisioned_reads))
return min_provisioned_reads
return updated_provisioning
def increase_writes_in_units(current_provisioning, units,
max_provisioned_writes, log_tag):
""" Increase the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
if int(units) > int(current_provisioning):
updated_provisioning = 2 * int(current_provisioning)
else:
updated_provisioning = int(current_provisioning) + int(units)
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag, max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units, key_name):
""" Increase the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) + int(units)
logger.debug(
'Write provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_writes') > 0:
if (updated_provisioning >
get_table_option(key_name, 'max_provisioned_writes')):
logger.info('Reached provisioned writes max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_writes'))))
return get_table_option(key_name, 'max_provisioned_writes')
return updated_provisioning
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = __get_aws_metric(
table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_read_units = 0
try:
consumed_read_units_percent = int(
math.ceil(
float(consumed_read_units) /
float(dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)) * 100))
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format(
table_name, gsi_name, consumed_read_units_percent))
return consumed_read_units_percent
def get_throttled_by_consumed_write_percent(table_name,
gsi_name,
lookback_window_start=15):
""" Returns the number of throttled write events in percent of consumption
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled write events by consumption
"""
try:
metrics1 = __get_aws_metric(table_name, gsi_name,
lookback_window_start,
'ConsumedWriteCapacityUnits')
metrics2 = __get_aws_metric(table_name, gsi_name,
lookback_window_start,
'WriteThrottleEvents')
except BotoServerError:
raise
if metrics1 and metrics2:
throttled_by_consumed_write_percent = (
((float(metrics2[0]['Sum']) / float(300)) /
(float(metrics1[0]['Sum']) / float(300))) * 100)
else:
throttled_by_consumed_write_percent = 0
logger.info('{0} - GSI: {1} - Throttled write percent '
'by consumption: {2:.2f}%'.format(
table_name, gsi_name, throttled_by_consumed_write_percent))
return throttled_by_consumed_write_percent
def get_throttled_read_event_count(
table_name, gsi_name, lookback_window_start=15):
""" Returns the number of throttled read events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: How many seconds to look at
:returns: int -- Number of throttled read events
"""
try:
metrics = __get_aws_metric(
table_name, gsi_name, lookback_window_start, 'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = int(metrics[0]['Sum'])
else:
throttled_read_events = 0
logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format(
table_name, gsi_name, throttled_read_events))
return throttled_read_events
def __ensure_provisioning_reads(table_name, key_name):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:returns: (bool, int) -- update_needed, updated_read_units
"""
update_needed = False
updated_read_units = statistics.get_provisioned_read_units(table_name)
consumed_read_units_percent = \
statistics.get_consumed_read_units_percent(table_name)
if (consumed_read_units_percent == 0 and not
get_table_option(
key_name, 'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - Scaling down reads is not done when usage is at 0%'.format(
table_name))
elif (consumed_read_units_percent >=
get_table_option(key_name, 'reads_upper_threshold')):
if get_table_option(key_name, 'increase_reads_unit') == 'percent':
updated_provisioning = calculators.increase_reads_in_percent(
updated_read_units,
get_table_option(key_name, 'increase_reads_with'),
key_name)
else:
updated_provisioning = calculators.increase_reads_in_units(
updated_read_units,
get_table_option(key_name, 'increase_reads_with'),
key_name)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
elif (consumed_read_units_percent <=
get_table_option(key_name, 'reads_lower_threshold')):
if get_table_option(key_name, 'decrease_reads_unit') == 'percent':
updated_provisioning = calculators.decrease_reads_in_percent(
updated_read_units,
get_table_option(key_name, 'decrease_reads_with'),
key_name)
else:
updated_provisioning = calculators.decrease_reads_in_units(
updated_read_units,
get_table_option(key_name, 'decrease_reads_with'),
key_name)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
return update_needed, int(updated_read_units)
def get_consumed_write_units_percent(table_name, time_frame=300):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(
table_name, time_frame, 'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_table_write_units(table_name)) *
100))
except JSONResponseError:
raise
logger.info('{0} - Consumed write units: {1:d}%'.format(
table_name, consumed_write_units_percent))
return consumed_write_units_percent
def ensure_provisioning(table_name, key_name):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
"""
if get_global_option('circuit_breaker_url'):
if circuit_breaker.is_open():
logger.warning('Circuit breaker is OPEN!')
return None
read_update_needed, updated_read_units = __ensure_provisioning_reads(
table_name, key_name)
write_update_needed, updated_write_units = __ensure_provisioning_writes(
table_name, key_name)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
__update_throughput(
table_name, updated_read_units, updated_write_units, key_name)
else:
logger.info('{0} - No need to change provisioning'.format(table_name))
def get_throttled_write_event_count(table_name, gsi_name, time_frame=300):
""" Returns the number of throttled write events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of throttled write events
"""
try:
metrics = __get_aws_metric(
table_name, gsi_name, time_frame, 'WriteThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_write_events = int(metrics[0]['Sum'])
else:
throttled_write_events = 0
logger.info('{0} - GSI: {1} - Write throttle count: {2:d}'.format(
table_name, gsi_name, throttled_write_events))
return throttled_write_events
def decrease_reads_in_units(current_provisioning, units, min_provisioned_reads,
log_tag):
""" Decrease the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type min_provisioned_reads: int
:param min_provisioned_reads: Configured min provisioned reads
:type log_tag: str
:param log_tag: Prefix for the log
"""
updated_provisioning = int(current_provisioning) - int(units)
min_provisioned_reads = __get_min_reads(current_provisioning,
min_provisioned_reads, log_tag)
if updated_provisioning < min_provisioned_reads:
logger.info('{0} - Reached provisioned reads min limit: {1:d}'.format(
log_tag, min_provisioned_reads))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name):
""" Decrease the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) - int(units)
logger.debug(
'Write provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_writes = get_min_provisioned_writes(
table_name,
current_provisioning,
key_name)
if min_provisioned_writes > 0:
if updated_provisioning < min_provisioned_writes:
logger.info('Reached provisioned writes min limit: {0:d}'.format(
min_provisioned_writes))
return min_provisioned_writes
return updated_provisioning
def main():
""" Main function called from dynamic-dynamodb """
try:
if get_global_option("daemon"):
daemon = DynamicDynamoDBDaemon(
"{0}/dynamic-dynamodb.{1}.pid".format(get_global_option("pid_file_dir"), get_global_option("instance"))
)
if get_global_option("daemon") == "start":
daemon.start()
elif get_global_option("daemon") == "stop":
logger.debug("Stopping daemon")
daemon.stop()
logger.info("Daemon stopped")
sys.exit(0)
elif get_global_option("daemon") == "restart":
daemon.restart()
elif get_global_option("daemon") in ["foreground", "fg"]:
daemon.run()
else:
print("Valid options for --daemon are start, stop and restart")
sys.exit(1)
else:
while True:
execute()
except Exception as error:
logger.exception(error)
def ensure_provisioning(table_name, key_name, num_consec_read_checks,
num_consec_write_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option('circuit_breaker_url') or get_table_option(
key_name, 'circuit_breaker_url'):
if circuit_breaker.is_open(table_name, key_name):
logger.warning('Circuit breaker is OPEN!')
return (0, 0)
# Handle throughput alarm checks
__ensure_provisioning_alarm(table_name, key_name)
try:
read_update_needed, updated_read_units, num_consec_read_checks = \
__ensure_provisioning_reads(
table_name,
key_name,
num_consec_read_checks)
write_update_needed, updated_write_units, num_consec_write_checks = \
__ensure_provisioning_writes(
table_name,
key_name,
num_consec_write_checks)
if read_update_needed:
num_consec_read_checks = 0
if write_update_needed:
num_consec_write_checks = 0
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info('{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name, int(updated_read_units),
int(updated_write_units)))
__update_throughput(table_name, key_name, updated_read_units,
updated_write_units)
else:
logger.info(
'{0} - No need to change provisioning'.format(table_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
def main():
""" Main function called from dynamic-dynamodb """
try:
if get_global_option('show_config'):
print json.dumps(config.get_configuration(), indent=2)
elif get_global_option('daemon'):
daemon = DynamicDynamoDBDaemon(
'{0}/dynamic-dynamodb.{1}.pid'.format(
get_global_option('pid_file_dir'),
get_global_option('instance')))
if get_global_option('daemon') == 'start':
logger.debug('Starting daemon')
try:
daemon.start()
logger.info('Daemon started')
except IOError as error:
logger.error(
'Could not create pid file: {0}'.format(error))
logger.error('Daemon not started')
elif get_global_option('daemon') == 'stop':
logger.debug('Stopping daemon')
daemon.stop()
logger.info('Daemon stopped')
sys.exit(0)
elif get_global_option('daemon') == 'restart':
logger.debug('Restarting daemon')
daemon.restart()
logger.info('Daemon restarted')
elif get_global_option('daemon') in ['foreground', 'fg']:
logger.debug('Starting daemon in foreground')
daemon.run()
logger.info('Daemon started in foreground')
else:
print(
'Valid options for --daemon are start, '
'stop, restart, and foreground')
sys.exit(1)
else:
if get_global_option('run_once'):
execute()
else:
while True:
execute()
except Exception as error:
logger.exception(error)
except KeyboardInterrupt:
while threading.active_count() > 1:
for thread in table_threads.values():
thread.do_run = False
logger.info('Waiting for all threads... {}s'.format(
get_global_option('check_interval')))
time.sleep(get_global_option('check_interval'))
raise
def __ensure_provisioning_writes(table_name):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:returns: (bool, int) -- update_needed, updated_write_units
"""
update_needed = False
updated_write_units = statistics.get_provisioned_write_units(table_name)
consumed_write_units_percent = \
statistics.get_consumed_write_units_percent(table_name)
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not
get_table_option(table_name, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - Scaling down writes is not done when usage is at 0%'.format(
table_name))
elif (consumed_write_units_percent >=
get_table_option(table_name, 'writes_upper_threshold')):
if get_table_option(table_name, 'increase_writes_unit') == 'percent':
updated_provisioning = calculators.increase_writes_in_percent(
table_name,
updated_write_units,
get_table_option(table_name, 'increase_writes_with'))
else:
updated_provisioning = calculators.increase_writes_in_units(
table_name,
updated_write_units,
get_table_option(table_name, 'increase_writes_with'))
update_needed = True
updated_write_units = updated_provisioning
elif (consumed_write_units_percent <=
get_table_option(table_name, 'writes_lower_threshold')):
if get_table_option(table_name, 'decrease_writes_unit') == 'percent':
updated_provisioning = calculators.decrease_writes_in_percent(
table_name,
updated_write_units,
get_table_option(table_name, 'decrease_writes_with'))
else:
updated_provisioning = calculators.decrease_writes_in_units(
table_name,
updated_write_units,
get_table_option(table_name, 'decrease_writes_with'))
update_needed = True
updated_write_units = updated_provisioning
return update_needed, int(updated_write_units)
def ensure_provisioning(
table_name, key_name,
num_consec_read_checks,
num_consec_write_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option('circuit_breaker_url'):
if circuit_breaker.is_open():
logger.warning('Circuit breaker is OPEN!')
return (0, 0)
try:
read_update_needed, updated_read_units, num_consec_read_checks = \
__ensure_provisioning_reads(
table_name,
key_name,
num_consec_read_checks)
write_update_needed, updated_write_units, num_consec_write_checks = \
__ensure_provisioning_writes(
table_name,
key_name,
num_consec_write_checks)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
__update_throughput(
table_name,
key_name,
updated_read_units,
updated_write_units)
else:
logger.info('{0} - No need to change provisioning'.format(
table_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units,
write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
current_wu = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
except JSONResponseError:
raise
# Check table status
try:
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
except JSONResponseError:
raise
logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
table_name, gsi_name, gsi_status))
if gsi_status != 'ACTIVE':
logger.warning(
'{0} - GSI: {1} - Not performing throughput changes when GSI '
'status is {2}'.format(table_name, gsi_name, gsi_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name, gsi_name, read_units, current_ru, write_units,
current_wu)
if read_units == current_ru and write_units == current_wu:
logger.info('{0} - GSI: {1} - No changes to perform'.format(
table_name, gsi_name))
return
dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key,
int(read_units), int(write_units))
def increase_reads_in_units(
current_provisioning, units, table_name, table_key, gsi_name, gsi_key):
""" Increase the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Name of the key
:returns: int -- New provisioning value
"""
updated_provisioning = 0
if int(units) > int(current_provisioning):
updated_provisioning = 2 * int(current_provisioning)
else:
updated_provisioning = int(current_provisioning) + int(units)
if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'):
max_provisioned_reads = int(get_gsi_option(
table_key, gsi_key, 'max_provisioned_reads'))
else:
max_provisioned_reads = 0
if (max_provisioned_reads > 0 and
updated_provisioning > max_provisioned_reads):
logger.info(
'{0} - GSI: {1} - '
'Reached provisioned reads max limit: {2:d}'.format(
table_name,
gsi_name,
max_provisioned_reads))
return max_provisioned_reads
logger.debug(
'{0} - GSI: {1} - '
'Read provisioning will be increased to {2:d} units'.format(
table_name,
gsi_name,
updated_provisioning))
return updated_provisioning
def __update_throughput(table_name, key_name, read_units, write_units):
""" Update throughput on the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_table_read_units(table_name)
current_wu = dynamodb.get_provisioned_table_write_units(table_name)
except JSONResponseError:
raise
# Check table status
try:
table_status = dynamodb.get_table_status(table_name)
except JSONResponseError:
raise
logger.debug('{0} - Table status is {1}'.format(table_name, table_status))
if table_status != 'ACTIVE':
logger.warning(
'{0} - Not performing throughput changes when table '
'is {1}'.format(table_name, table_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_table_option(key_name, 'always_decrease_rw_together'):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name,
read_units,
current_ru,
write_units,
current_wu)
if read_units == current_ru and write_units == current_wu:
logger.info('{0} - No changes to perform'.format(table_name))
return
dynamodb.update_table_provisioning(
table_name,
key_name,
int(read_units),
int(write_units))
def execute_in_thread(table_name, table_key):
if table_name in table_threads:
thread = table_threads.get(table_name)
if thread.isAlive():
logger.debug("Thread {} still running".format(table_name))
else:
logger.error("Thread {} was stopped!".format(table_name))
sys.exit(1)
else:
table_logger = log_handler.getTableLogger(table_name)
logger.info("Start new thread: " + table_name)
thread = threading.Thread(target=execute_table_in_loop,
args=[table_name, table_key, table_logger])
table_threads[table_name] = thread
thread.start()
def increase_reads_in_percent(current_provisioning, percent,
max_provisioned_reads,
consumed_read_units_percent, log_tag):
""" Increase the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:type max_provisioned_reads: int
:param max_provisioned_reads: Configured max provisioned reads
:returns: int -- New provisioning value
:type consumed_read_units_percent: float
:param consumed_read_units_percent: Number of consumed read units
:type log_tag: str
:param log_tag: Prefix for the log
"""
current_provisioning = float(current_provisioning)
consumed_read_units_percent = float(consumed_read_units_percent)
percent = float(percent)
consumption_based_current_provisioning = \
float(math.ceil(current_provisioning*(consumed_read_units_percent/100)))
if consumption_based_current_provisioning > current_provisioning:
increase = int(
math.ceil(consumption_based_current_provisioning *
(percent / 100)))
updated_provisioning = consumption_based_current_provisioning + increase
else:
increase = int(math.ceil(current_provisioning * (percent / 100)))
updated_provisioning = current_provisioning + increase
if max_provisioned_reads > 0:
if updated_provisioning > max_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads max limit: {1}'.format(
log_tag, max_provisioned_reads))
return max_provisioned_reads
logger.debug(
'{0} - Read provisioning will be increased to {1} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def __calculate_always_decrease_rw_values(table_name, gsi_name, read_units,
provisioned_reads, write_units,
provisioned_writes):
""" Calculate values for always-decrease-rw-together
This will only return reads and writes decreases if both reads and writes
are lower than the current provisioning
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type read_units: int
:param read_units: New read unit provisioning
:type provisioned_reads: int
:param provisioned_reads: Currently provisioned reads
:type write_units: int
:param write_units: New write unit provisioning
:type provisioned_writes: int
:param provisioned_writes: Currently provisioned writes
:returns: (int, int) -- (reads, writes)
"""
if read_units < provisioned_reads and write_units < provisioned_writes:
return (read_units, write_units)
if read_units < provisioned_reads:
logger.info(
'{0} - GSI: {1} - Reads could be decreased, '
'but we are waiting for writes to get lower than the threshold '
'before scaling down'.format(table_name, gsi_name))
read_units = provisioned_reads
elif write_units < provisioned_writes:
logger.info(
'{0} - GSI: {1} - Writes could be decreased, '
'but we are waiting for reads to get lower than the threshold '
'before scaling down'.format(table_name, gsi_name))
write_units = provisioned_writes
return (read_units, write_units)
def get_throttled_by_provisioned_read_event_percent(table_name,
gsi_name,
lookback_window_start=15,
lookback_period=5):
""" Returns the number of throttled read events in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: float -- Percent of throttled read events by provisioning
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, lookback_window_start,
lookback_period, 'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
lookback_seconds = lookback_period * 60
throttled_read_events = (float(metrics[0]['Sum']) /
float(lookback_seconds))
else:
throttled_read_events = 0
try:
gsi_read_units = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
throttled_by_provisioned_read_percent = (float(throttled_read_events) /
float(gsi_read_units) * 100)
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Throttled read percent '
'by provision: {2:.2f}%'.format(
table_name, gsi_name,
throttled_by_provisioned_read_percent))
return throttled_by_provisioned_read_percent
def __publish(topic, message, subject=None):
""" Publish a message to a SNS topic
:type topic: str
:param topic: SNS topic to publish the message to
:type message: str
:param message: Message to send via SNS
:type subject: str
:param subject: Subject to use for e-mail notifications
:returns: None
"""
try:
SNS_CONNECTION.publish(topic=topic, message=message, subject=subject)
logger.info('Sent SNS notification to {0}'.format(topic))
except BotoServerError as error:
logger.error('Problem sending SNS notification: {0}'.format(
error.message))
return
def get_consumed_write_units_percent(table_name,
gsi_name,
lookback_window_start=15,
lookback_period=5):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: float -- Number of consumed writes as a
percentage of provisioned writes
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, lookback_window_start,
lookback_period,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
lookback_seconds = lookback_period * 60
consumed_write_units = (float(metrics[0]['Sum']) /
float(lookback_seconds))
else:
consumed_write_units = 0
try:
gsi_write_units = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
consumed_write_units_percent = (float(consumed_write_units) /
float(gsi_write_units) * 100)
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed write units: {2:.2f}%'.format(
table_name, gsi_name, consumed_write_units_percent))
return consumed_write_units_percent
def increase_writes_in_units(current_provisioning, units,
max_provisioned_writes,
consumed_write_units_percent, log_tag):
""" Increase the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type consumed_write_units_percent: float
:param consumed_write_units_percent: Number of consumed write units
:type log_tag: str
:param log_tag: Prefix for the log
"""
units = int(units)
current_provisioning = float(current_provisioning)
consumed_write_units_percent = float(consumed_write_units_percent)
consumption_based_current_provisioning = \
int(math.ceil(current_provisioning*(consumed_write_units_percent/100)))
if consumption_based_current_provisioning > current_provisioning:
updated_provisioning = consumption_based_current_provisioning + units
else:
updated_provisioning = int(current_provisioning) + units
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag, max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag, int(updated_provisioning)))
return updated_provisioning
def main():
""" Main function called from dynamic-dynamodb """
try:
if get_global_option('show_config'):
print json.dumps(config.get_configuration(), indent=2)
elif get_global_option('daemon'):
daemon = DynamicDynamoDBDaemon(
'{0}/dynamic-dynamodb.{1}.pid'.format(
get_global_option('pid_file_dir'),
get_global_option('instance')))
if get_global_option('daemon') == 'start':
logger.debug('Starting daemon')
try:
daemon.start()
logger.info('Daemon started')
except IOError as error:
logger.error(
'Could not create pid file: {0}'.format(error))
logger.error('Daemon not started')
elif get_global_option('daemon') == 'stop':
logger.debug('Stopping daemon')
daemon.stop()
logger.info('Daemon stopped')
sys.exit(0)
elif get_global_option('daemon') == 'restart':
logger.debug('Restarting daemon')
daemon.restart()
logger.info('Daemon restarted')
elif get_global_option('daemon') in ['foreground', 'fg']:
logger.debug('Starting daemon in foreground')
daemon.run()
logger.info('Daemon started in foreground')
else:
print(
'Valid options for --daemon are start, '
'stop, restart, and foreground')
sys.exit(1)
else:
if get_global_option('run_once'):
execute()
else:
while True:
execute()
except Exception as error:
logger.exception(error)
