def get_min_provisioned_reads(current_provisioning, table_name, key_name):
""" Returns the minimum provisioned reads
If the min_provisioned_reads value is less than current_provisioning * 2,
then we return current_provisioning * 2, as DynamoDB cannot be scaled up
with more than 100%.
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Name of the key
:returns: int -- Minimum provisioned reads
"""
min_provisioned_reads = 1
if get_table_option(key_name, 'min_provisioned_reads'):
min_provisioned_reads = int(
get_table_option(key_name, 'min_provisioned_reads'))
if min_provisioned_reads > int(current_provisioning * 2):
min_provisioned_reads = int(current_provisioning * 2)
logger.debug(
'{0} - '
'Cannot reach min_provisioned_reads as max scale up '
'is 100% of current provisioning'.format(
table_name))
logger.debug(
'{0} - '
'Setting min provisioned reads to {1}'.format(
table_name, min_provisioned_reads))
return min_provisioned_reads
def decrease_reads_in_percent(
current_provisioning, percent, min_provisioned_reads, log_tag):
""" Decrease the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:type min_provisioned_reads: int
:param min_provisioned_reads: Configured min provisioned reads
:type log_tag: str
:param log_tag: Prefix for the log
:returns: int -- New provisioning value
"""
percent = float(percent)
decrease = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning - decrease
min_provisioned_reads = __get_min_reads(
current_provisioning,
min_provisioned_reads,
log_tag)
if updated_provisioning < min_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads min limit: {1:d}'.format(
log_tag, int(min_provisioned_reads)))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
log_tag, int(updated_provisioning)))
return updated_provisioning
def decrease_writes_in_percent(
current_provisioning, percent, key_name, table_name):
""" Decrease the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
:type table_name: str
:param table_name: Name of the DynamoDB table
"""
decrease = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning - decrease
min_provisioned_writes = get_min_provisioned_writes(
current_provisioning,
table_name,
key_name)
if updated_provisioning < min_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes min limit: {1:d}'.format(
table_name,
min_provisioned_writes))
return min_provisioned_writes
logger.debug(
'{0} - Write provisioning will be decreased to {1:d} units'.format(
table_name,
updated_provisioning))
return updated_provisioning
def get_consumed_read_units_percent(table_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
period=time_frame,
start_time=datetime.utcnow()-timedelta(minutes=10, seconds=time_frame),
end_time=datetime.utcnow()-timedelta(minutes=10),
metric_name='ConsumedReadCapacityUnits',
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_read_units = 0
consumed_read_units_percent = int(math.ceil(
float(consumed_read_units) / \
float(get_provisioned_read_units(table_name)) * \
100))
logger.info('{0} - Consumed read units: {1:d}%'.format(
table_name, consumed_read_units_percent))
return consumed_read_units_percent
def get_consumed_read_units(table_name, time_frame=300):
""" Returns the number of consumed read units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
time_frame,
datetime.utcnow()-timedelta(minutes=10, seconds=time_frame),
datetime.utcnow()-timedelta(minutes=10),
'ConsumedReadCapacityUnits',
'AWS/DynamoDB',
['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_read_units = 0
logger.info('{0} - Consumed read units: {1:d}'.format(
table_name, consumed_read_units))
return consumed_read_units
def run(self, check_interval=1):
""" Run the daemon
:type check_interval: int
:param check_interval: Delay in seconds between checks
"""
while True:
used_keys = set()
table_names = set()
configured_tables = configuration['tables'].keys()
# Add regexp table names
for table_name in core.dynamodb.list_tables():
for key_name in configured_tables:
if re.match(key_name, table_name):
logger.debug(
"Table {0} match with config key {1}".format(
table_name, key_name))
table_names.add((table_name, key_name))
used_keys.add(key_name)
# Remove used tables
for table_name in used_keys:
configured_tables.remove(table_name)
# Ensure provisioning
for table_name, key_name in sorted(table_names):
core.ensure_provisioning(table_name, key_name)
# Sleep between the checks
time.sleep(check_interval)
def decrease_reads_in_units(
current_provisioning, units, min_provisioned_reads, log_tag):
""" Decrease the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type min_provisioned_reads: int
:param min_provisioned_reads: Configured min provisioned reads
:type log_tag: str
:param log_tag: Prefix for the log
"""
updated_provisioning = int(current_provisioning) - int(units)
min_provisioned_reads = __get_min_reads(
current_provisioning,
min_provisioned_reads,
log_tag)
if updated_provisioning < min_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads min limit: {1:d}'.format(
log_tag,
int(min_provisioned_reads)))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
log_tag,
int(updated_provisioning)))
return updated_provisioning
def decrease_writes_in_percent(current_provisioning, percent,
min_provisioned_writes, log_tag):
""" Decrease the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type min_provisioned_writes: int
:param min_provisioned_writes: Configured min provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
decrease = int(float(current_provisioning) * (float(percent) / 100))
updated_provisioning = current_provisioning - decrease
min_provisioned_writes = __get_min_writes(current_provisioning,
min_provisioned_writes, log_tag)
if updated_provisioning < min_provisioned_writes:
logger.info('{0} - Reached provisioned writes min limit: {1:d}'.format(
log_tag, min_provisioned_writes))
return min_provisioned_writes
logger.debug(
'{0} - Write provisioning will be decreased to {1:d} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def get_throttled_by_provisioned_read_event_percent(table_name,
lookback_window_start=15):
""" Returns the number of throttled read events in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled read events by provisioning
"""
try:
metrics = __get_aws_metric(table_name, lookback_window_start,
'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = float(metrics[0]['Sum']) / float(300)
else:
throttled_read_events = 0
try:
throttled_by_provisioned_read_percent = (
float(throttled_read_events) /
float(dynamodb.get_provisioned_table_read_units(table_name)) * 100)
except JSONResponseError:
raise
logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format(
table_name, throttled_by_provisioned_read_percent))
return throttled_by_provisioned_read_percent
def get_throttled_by_consumed_write_percent(table_name, lookback_window_start=15):
""" Returns the number of throttled write events in percent of consumption
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled write events by consumption
"""
try:
metrics1 = __get_aws_metric(
table_name, lookback_window_start, 'ConsumedWriteCapacityUnits')
metrics2 = __get_aws_metric(
table_name, lookback_window_start, 'WriteThrottleEvents')
except BotoServerError:
raise
if metrics1 and metrics2:
throttled_by_consumed_write_percent = (((float(metrics2[0]['Sum'])/float(300)) /
(float(metrics1[0]['Sum'])/float(300))) * 100)
else:
throttled_by_consumed_write_percent = 0
logger.info('{0} - Throttled write percent by consumption: {1:.2f}%'.format(
table_name, throttled_by_consumed_write_percent))
return throttled_by_consumed_write_percent
def __get_min_writes(current_provisioning, min_provisioned_writes, log_tag):
""" Get the minimum number of writes to current_provisioning
:type current_provisioning: int
:param current_provisioning: Current provisioned writes
:type min_provisioned_writes: int
:param min_provisioned_writes: Configured min provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
:returns: int -- Minimum number of writes
"""
# Fallback value to ensure that we always have at least 1 read
writes = 1
if min_provisioned_writes:
writes = int(min_provisioned_writes)
if writes > int(current_provisioning * 2):
writes = int(current_provisioning * 2)
logger.debug('{0} - '
'Cannot reach min-provisioned-writes as max scale up '
'is 100% of current provisioning'.format(log_tag))
logger.debug('{0} - Setting min provisioned writes to {1}'.format(
log_tag, min_provisioned_writes))
return writes
def decrease_reads_in_percent(table_name, current_provisioning, percent,
key_name):
""" Decrease the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
decrease = int(float(current_provisioning) * (float(percent) / 100))
updated_provisioning = current_provisioning - decrease
logger.debug('Read provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_reads = get_min_provisioned_reads(table_name,
current_provisioning,
key_name)
if min_provisioned_reads > 0:
if updated_provisioning < min_provisioned_reads:
logger.info('Reached provisioned reads min limit: {0:d}'.format(
min_provisioned_reads))
return min_provisioned_reads
return updated_provisioning
def increase_reads_in_percent(table_name, current_provisioning, percent,
key_name):
""" Increase the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
increase = int(float(current_provisioning) * (float(percent) / 100))
updated_provisioning = current_provisioning + increase
logger.debug('Read provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_reads') > 0:
if (updated_provisioning > get_table_option(key_name,
'max_provisioned_reads')):
logger.info('Reached provisioned reads max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_reads'))))
return get_table_option(key_name, 'max_provisioned_reads')
return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units,
key_name):
""" Increase the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) + int(units)
logger.debug('Write provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_writes') > 0:
if (updated_provisioning > get_table_option(key_name,
'max_provisioned_writes')):
logger.info('Reached provisioned writes max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_writes'))))
return get_table_option(key_name, 'max_provisioned_writes')
return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units,
key_name):
""" Decrease the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) - int(units)
logger.debug('Write provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_writes = get_min_provisioned_writes(
table_name, current_provisioning, key_name)
if min_provisioned_writes > 0:
if updated_provisioning < min_provisioned_writes:
logger.info('Reached provisioned writes min limit: {0:d}'.format(
min_provisioned_writes))
return min_provisioned_writes
return updated_provisioning
def get_consumed_read_units(table_name, time_frame=300):
""" Returns the number of consumed read units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
time_frame,
datetime.utcnow() - timedelta(minutes=10, seconds=time_frame),
datetime.utcnow() - timedelta(minutes=10),
'ConsumedReadCapacityUnits',
'AWS/DynamoDB', ['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_read_units = 0
logger.info('{0} - Consumed read units: {1:d}'.format(
table_name, consumed_read_units))
return consumed_read_units
def get_throttled_read_event_count(
table_name, lookback_window_start=15, lookback_period=5):
""" Returns the number of throttled read events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:type lookback_period: int
:param lookback_period: Number of minutes to look at
:returns: int -- Number of throttled read events during the time period
"""
try:
metrics = __get_aws_metric(
table_name,
lookback_window_start,
lookback_period,
'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = int(metrics[0]['Sum'])
else:
throttled_read_events = 0
logger.info('{0} - Read throttle count: {1:d}'.format(
table_name, throttled_read_events))
return throttled_read_events
def get_throttled_by_provisioned_read_event_percent(table_name, lookback_window_start=15):
""" Returns the number of throttled read events in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled read events by provisioning
"""
try:
metrics = __get_aws_metric(
table_name, lookback_window_start, 'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = float(metrics[0]['Sum'])/float(300)
else:
throttled_read_events = 0
try: throttled_by_provisioned_read_percent = (float(throttled_read_events) /
float(dynamodb.get_provisioned_table_read_units(table_name)) *
100)
except JSONResponseError:
raise
logger.info('{0} - Throttled read percent by provision: {1:.2f}%'.format(
table_name, throttled_by_provisioned_read_percent))
return throttled_by_provisioned_read_percent
def ensure_created(table_name, template_table_name):
""" Ensure table has been created in DynamoDB based on given template table name
:type table_name: str
:param table_name: Name of the DynamoDB table
:type template_table_name: str
:param template_table_name: Name of the template DynamoDB table (that has hashkey, attribute definitions)
"""
try:
desc = DYNAMODB_CONNECTION.describe_table(table_name)[u'Table']
except JSONResponseError:
try :
template_table = get_table( template_table_name )
template_table.describe()
logger.info(
'{0} - Create table with template table schema {1}, throughput {2}, indexes {3}, global_indexes {4}'.format(
table_name, template_table.schema, template_table.throughput, template_table.indexes, template_table.global_indexes))
# Return if dry-run
if get_global_option('dry_run'):
return
table = Table.create(table_name, schema=template_table.schema,
throughput=template_table.throughput, indexes=template_table.indexes,
global_indexes=template_table.global_indexes,
connection=DYNAMODB_CONNECTION)
except DynamoDBResponseError:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'ResourceNotFoundException':
logger.error(
'{0} - Table {1} not found'.format(table_name, table_name))
raise
def get_throttled_by_consumed_write_percent(table_name,
lookback_window_start=15):
""" Returns the number of throttled write events in percent of consumption
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: float -- Percent of throttled write events by consumption
"""
try:
metrics1 = __get_aws_metric(table_name, lookback_window_start,
'ConsumedWriteCapacityUnits')
metrics2 = __get_aws_metric(table_name, lookback_window_start,
'WriteThrottleEvents')
except BotoServerError:
raise
if metrics1 and metrics2:
throttled_by_consumed_write_percent = (
((float(metrics2[0]['Sum']) / float(300)) /
(float(metrics1[0]['Sum']) / float(300))) * 100)
else:
throttled_by_consumed_write_percent = 0
logger.info(
'{0} - Throttled write percent by consumption: {1:.2f}%'.format(
table_name, throttled_by_consumed_write_percent))
return throttled_by_consumed_write_percent
def decrease_reads_in_units(current_provisioning, units, min_provisioned_reads,
log_tag):
""" Decrease the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type min_provisioned_reads: int
:param min_provisioned_reads: Configured min provisioned reads
:type log_tag: str
:param log_tag: Prefix for the log
"""
updated_provisioning = int(current_provisioning) - int(units)
min_provisioned_reads = __get_min_reads(current_provisioning,
min_provisioned_reads, log_tag)
if updated_provisioning < min_provisioned_reads:
logger.info('{0} - Reached provisioned reads min limit: {1:d}'.format(
log_tag, min_provisioned_reads))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
log_tag, updated_provisioning))
return updated_provisioning
def __is_table_maintenance_window(table_name, maintenance_windows):
""" Checks that the current time is within the maintenance window
:type table_name: str
:param table_name: Name of the DynamoDB table
:type maintenance_windows: str
:param maintenance_windows: Example: '00:00-01:00,10:00-11:00'
:returns: bool -- True if within maintenance window
"""
# Example string '00:00-01:00,10:00-11:00'
maintenance_window_list = []
for window in maintenance_windows.split(','):
try:
start, end = window.split('-', 1)
except ValueError:
logger.error(
'{0} - Malformatted maintenance window'.format(table_name))
return False
maintenance_window_list.append((start, end))
now = datetime.datetime.utcnow().strftime('%H%M')
for maintenance_window in maintenance_window_list:
start = ''.join(maintenance_window[0].split(':'))
end = ''.join(maintenance_window[1].split(':'))
if now >= start and now <= end:
return True
return False
def __is_maintenance_window(table_name, maintenance_windows):
""" Checks that the current time is within the maintenance window
:type table_name: str
:param table_name: Name of the DynamoDB table
:type maintenance_windows: str
:param maintenance_windows: Example: '00:00-01:00,10:00-11:00'
:returns: bool -- True if within maintenance window
"""
# Example string '00:00-01:00,10:00-11:00'
maintenance_window_list = []
for window in maintenance_windows.split(','):
try:
start, end = window.split('-', 1)
except ValueError:
logger.error(
'{0} - Malformatted maintenance window'.format(table_name))
return False
maintenance_window_list.append((start, end))
now = datetime.datetime.utcnow().strftime('%H%M')
for maintenance_window in maintenance_window_list:
start = ''.join(maintenance_window[0].split(':'))
end = ''.join(maintenance_window[1].split(':'))
if now >= start and now <= end:
return True
return False
def get_throttled_read_event_count(table_name, gsi_name, time_frame=300):
""" Returns the number of throttled read events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of throttled read events
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, time_frame,
'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = int(metrics[0]['Sum'])
else:
throttled_read_events = 0
logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format(
table_name, gsi_name, throttled_read_events))
return throttled_read_events
def increase_writes_in_percent(
current_provisioning, percent, max_provisioned_writes, log_tag):
""" Increase the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
increase = int(math.ceil(float(current_provisioning)*(float(percent)/100)))
updated_provisioning = current_provisioning + increase
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag,
max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag,
updated_provisioning))
return updated_provisioning
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
try:
metrics = __get_aws_metric(table_name, gsi_name, time_frame,
'ConsumedReadCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_read_units = 0
try:
consumed_read_units_percent = int(
math.ceil(
float(consumed_read_units) / float(
dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)) * 100))
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format(
table_name, gsi_name, consumed_read_units_percent))
return consumed_read_units_percent
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name):
""" Decrease the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
decrease = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning - decrease
logger.debug(
'Read provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_reads = get_min_provisioned_reads(
table_name,
current_provisioning,
key_name)
if min_provisioned_reads > 0:
if updated_provisioning < min_provisioned_reads:
logger.info('Reached provisioned reads min limit: {0:d}'.format(
min_provisioned_reads))
return min_provisioned_reads
return updated_provisioning
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name):
""" Increase the current_provisioning with percent %
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
increase = int(float(current_provisioning)*(float(percent)/100))
updated_provisioning = current_provisioning + increase
logger.debug(
'Read provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_reads') > 0:
if (updated_provisioning >
get_table_option(key_name, 'max_provisioned_reads')):
logger.info('Reached provisioned reads max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_reads'))))
return get_table_option(key_name, 'max_provisioned_reads')
return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name):
""" Decrease the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) - int(units)
logger.debug(
'Write provisioning will be decreased to {0:d} units'.format(
updated_provisioning))
min_provisioned_writes = get_min_provisioned_writes(
table_name,
current_provisioning,
key_name)
if min_provisioned_writes > 0:
if updated_provisioning < min_provisioned_writes:
logger.info('Reached provisioned writes min limit: {0:d}'.format(
min_provisioned_writes))
return min_provisioned_writes
return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units, key_name):
""" Increase the current_provisioning with units units
:type table_name: str
:param table_name: Name of the DynamoDB table
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
"""
updated_provisioning = int(current_provisioning) + int(units)
logger.debug(
'Write provisioning will be increased to {0:d} units'.format(
updated_provisioning))
if get_table_option(key_name, 'max_provisioned_writes') > 0:
if (updated_provisioning >
get_table_option(key_name, 'max_provisioned_writes')):
logger.info('Reached provisioned writes max limit: {0:d}'.format(
int(get_table_option(key_name, 'max_provisioned_writes'))))
return get_table_option(key_name, 'max_provisioned_writes')
return updated_provisioning
def __get_aws_metric(table_name, lookback_window_start, metric_name):
""" Returns a metric list from the AWS CloudWatch service, may return
None if no metric exists
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: How many minutes to look at
:type metric_name: str
:param metric_name: Name of the metric to retrieve from CloudWatch
:returns: list -- A list of time series data for the given metric, may
be None if there was no data
"""
try:
now = datetime.utcnow()
start_time = now - timedelta(minutes=lookback_window_start)
end_time = now - timedelta(minutes=lookback_window_start - 5)
return cloudwatch_connection.get_metric_statistics(
period=300, # Always look at 5 minutes windows
start_time=start_time,
end_time=end_time,
metric_name=metric_name,
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': table_name},
unit='Count')
except BotoServerError as error:
logger.error('Unknown boto error. Status: "{0}". '
'Reason: "{1}". Message: {2}'.format(
error.status, error.reason, error.message))
raise
def get_consumed_read_units_percent(table_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = cloudwatch_connection.get_metric_statistics(
period=time_frame,
start_time=datetime.utcnow() -
timedelta(minutes=10, seconds=time_frame),
end_time=datetime.utcnow() - timedelta(minutes=10),
metric_name='ConsumedReadCapacityUnits',
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': table_name},
unit='Count')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum']) / float(time_frame)))
else:
consumed_read_units = 0
consumed_read_units_percent = int(math.ceil(
float(consumed_read_units) / \
float(get_provisioned_read_units(table_name)) * \
100))
logger.info('{0} - Consumed read units: {1:d}%'.format(
table_name, consumed_read_units_percent))
return consumed_read_units_percent
def get_consumed_write_units_percent(table_name, lookback_window_start=15):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(
table_name, lookback_window_start, 'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum'])/float(300)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_table_write_units(table_name)) *
100))
except JSONResponseError:
raise
logger.info('{0} - Consumed write units: {1:d}%'.format(
table_name, consumed_write_units_percent))
return consumed_write_units_percent
def decrease_reads_in_units(current_provisioning, units, key_name, table_name):
""" Decrease the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we decrease with
:returns: int -- New provisioning value
:type key_name: str
:param key_name: Name of the key
:type table_name: str
:param table_name: Name of the DynamoDB table
"""
updated_provisioning = int(current_provisioning) - int(units)
min_provisioned_reads = get_min_provisioned_reads(
current_provisioning,
table_name,
key_name)
if updated_provisioning < min_provisioned_reads:
logger.info(
'{0} - Reached provisioned reads min limit: {1:d}'.format(
table_name,
min_provisioned_reads))
return min_provisioned_reads
logger.debug(
'{0} - Read provisioning will be decreased to {1:d} units'.format(
table_name,
updated_provisioning))
return updated_provisioning
def __get_min_writes(current_provisioning, min_provisioned_writes, log_tag):
""" Get the minimum number of writes to current_provisioning
:type current_provisioning: int
:param current_provisioning: Current provisioned writes
:type min_provisioned_writes: int
:param min_provisioned_writes: Configured min provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
:returns: int -- Minimum number of writes
"""
# Fallback value to ensure that we always have at least 1 read
writes = 1
if min_provisioned_writes:
writes = int(min_provisioned_writes)
if writes > int(current_provisioning * 2):
writes = int(current_provisioning * 2)
logger.debug(
'{0} - '
'Cannot reach min-provisioned-writes as max scale up '
'is 100% of current provisioning'.format(log_tag))
logger.debug(
'{0} - Setting min provisioned writes to {1}'.format(
log_tag, min_provisioned_writes))
return writes
def __ensure_provisioning_reads(table_name, key_name):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:returns: (bool, int) -- update_needed, updated_read_units
"""
update_needed = False
updated_read_units = statistics.get_provisioned_read_units(table_name)
consumed_read_units_percent = \
statistics.get_consumed_read_units_percent(table_name)
if (consumed_read_units_percent == 0 and not
get_table_option(
key_name, 'allow_scaling_down_reads_on_0_percent')):
logger.info(
'{0} - Scaling down reads is not done when usage is at 0%'.format(
table_name))
elif (consumed_read_units_percent >=
get_table_option(key_name, 'reads_upper_threshold')):
if get_table_option(key_name, 'increase_reads_unit') == 'percent':
updated_provisioning = calculators.increase_reads_in_percent(
updated_read_units,
get_table_option(key_name, 'increase_reads_with'),
key_name)
else:
updated_provisioning = calculators.increase_reads_in_units(
updated_read_units,
get_table_option(key_name, 'increase_reads_with'),
key_name)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
elif (consumed_read_units_percent <=
get_table_option(key_name, 'reads_lower_threshold')):
if get_table_option(key_name, 'decrease_reads_unit') == 'percent':
updated_provisioning = calculators.decrease_reads_in_percent(
updated_read_units,
get_table_option(key_name, 'decrease_reads_with'),
key_name)
else:
updated_provisioning = calculators.decrease_reads_in_units(
updated_read_units,
get_table_option(key_name, 'decrease_reads_with'),
key_name)
if updated_read_units != updated_provisioning:
update_needed = True
updated_read_units = updated_provisioning
return update_needed, int(updated_read_units)
def increase_writes_in_units(
current_provisioning, units, max_provisioned_writes, log_tag):
""" Increase the current_provisioning with units units
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type units: int
:param units: How many units should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type log_tag: str
:param log_tag: Prefix for the log
"""
if int(units) > int(current_provisioning):
updated_provisioning = 2 * int(current_provisioning)
else:
updated_provisioning = int(current_provisioning) + int(units)
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag,
max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag,
updated_provisioning))
return updated_provisioning
def get_consumed_read_units_percent(table_name, gsi_name, time_frame=300):
""" Returns the number of consumed read units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed reads
"""
metrics = __get_aws_metric(
table_name, gsi_name, time_frame, 'ConsumedReadCapacityUnits')
if metrics:
consumed_read_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_read_units = 0
try:
consumed_read_units_percent = int(
math.ceil(
float(consumed_read_units) /
float(dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)) * 100))
except JSONResponseError:
raise
logger.info('{0} - GSI: {1} - Consumed read units: {2:d}%'.format(
table_name, gsi_name, consumed_read_units_percent))
return consumed_read_units_percent
def __get_connection_cloudwatch():
""" Ensure connection to CloudWatch """
region = get_global_option('region')
try:
if (get_global_option('aws_access_key_id')
and get_global_option('aws_secret_access_key')):
logger.debug('Authenticating to CloudWatch using '
'credentials in configuration file')
connection = cloudwatch.connect_to_region(
region,
aws_access_key_id=get_global_option('aws_access_key_id'),
aws_secret_access_key=get_global_option(
'aws_secret_access_key'))
else:
logger.debug('Authenticating using boto\'s authentication handler')
connection = cloudwatch.connect_to_region(region)
except Exception as err:
logger.error('Failed connecting to CloudWatch: {0}'.format(err))
logger.error('Please report an issue at: '
'https://github.com/sebdah/dynamic-dynamodb/issues')
raise
logger.debug('Connected to CloudWatch in {0}'.format(region))
return connection
def get_consumed_write_units_percent(table_name, lookback_window_start=15):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type lookback_window_start: int
:param lookback_window_start: Relative start time for the CloudWatch metric
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(table_name, lookback_window_start,
'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum']) / float(300)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_table_write_units(table_name)) *
100))
except JSONResponseError:
raise
logger.info('{0} - Consumed write units: {1:d}%'.format(
table_name, consumed_write_units_percent))
return consumed_write_units_percent
def get_consumed_write_units_percent(table_name, time_frame=300):
""" Returns the number of consumed write units in percent
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of consumed writes
"""
try:
metrics = __get_aws_metric(
table_name, time_frame, 'ConsumedWriteCapacityUnits')
except BotoServerError:
raise
if metrics:
consumed_write_units = int(
math.ceil(float(metrics[0]['Sum'])/float(time_frame)))
else:
consumed_write_units = 0
try:
consumed_write_units_percent = int(
math.ceil(
float(consumed_write_units) /
float(dynamodb.get_provisioned_table_write_units(table_name)) *
100))
except JSONResponseError:
raise
logger.info('{0} - Consumed write units: {1:d}%'.format(
table_name, consumed_write_units_percent))
return consumed_write_units_percent
def get_throttled_read_event_count(
table_name, gsi_name, lookback_window_start=15):
""" Returns the number of throttled read events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type lookback_window_start: int
:param lookback_window_start: How many seconds to look at
:returns: int -- Number of throttled read events
"""
try:
metrics = __get_aws_metric(
table_name, gsi_name, lookback_window_start, 'ReadThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_read_events = int(metrics[0]['Sum'])
else:
throttled_read_events = 0
logger.info('{0} - GSI: {1} - Read throttle count: {2:d}'.format(
table_name, gsi_name, throttled_read_events))
return throttled_read_events
def get_throttled_write_event_count(table_name, gsi_name, time_frame=300):
""" Returns the number of throttled write events during a given time frame
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type time_frame: int
:param time_frame: How many seconds to look at
:returns: int -- Number of throttled write events
"""
try:
metrics = __get_aws_metric(
table_name, gsi_name, time_frame, 'WriteThrottleEvents')
except BotoServerError:
raise
if metrics:
throttled_write_events = int(metrics[0]['Sum'])
else:
throttled_write_events = 0
logger.info('{0} - GSI: {1} - Write throttle count: {2:d}'.format(
table_name, gsi_name, throttled_write_events))
return throttled_write_events
def __get_aws_metric(table_name, time_frame, metric_name):
""" Returns a metric list from the AWS CloudWatch service, may return
None if no metric exists
:type table_name: str
:param table_name: Name of the DynamoDB table
:type time_frame: int
:param time_frame: How many seconds to look at
:type metric_name str
:param metric_name Name of the metric to retrieve from CloudWatch
:returns: list -- A list of time series data for the given metric, may
be None if there was no data
"""
try:
start_time = datetime.utcnow()-timedelta(minutes=10, seconds=time_frame)
end_time = datetime.utcnow()-timedelta(minutes=10)
return cloudwatch_connection.get_metric_statistics(
period=time_frame,
start_time=start_time,
end_time=end_time,
metric_name=metric_name,
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': table_name},
unit='Count')
except BotoServerError as error:
logger.error(
'Unknown boto error. Status: "{0}". '
'Reason: "{1}". Message: {2}'.format(
error.status,
error.reason,
error.message))
raise
def __get_connection_cloudwatch():
""" Ensure connection to CloudWatch """
region = get_global_option('region')
try:
if (get_global_option('aws_access_key_id') and
get_global_option('aws_secret_access_key')):
logger.debug(
'Authenticating to CloudWatch using '
'credentials in configuration file')
connection = cloudwatch.connect_to_region(
region,
aws_access_key_id=get_global_option('aws_access_key_id'),
aws_secret_access_key=get_global_option(
'aws_secret_access_key'))
else:
logger.debug(
'Authenticating using boto\'s authentication handler')
connection = cloudwatch.connect_to_region(region)
except Exception as err:
logger.error('Failed connecting to CloudWatch: {0}'.format(err))
logger.error(
'Please report an issue at: '
'https://github.com/sebdah/dynamic-dynamodb/issues')
raise
logger.debug('Connected to CloudWatch in {0}'.format(region))
return connection
def __ensure_provisioning_writes(table_name):
""" Ensure that provisioning of writes is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:returns: (bool, int) -- update_needed, updated_write_units
"""
update_needed = False
updated_write_units = statistics.get_provisioned_write_units(table_name)
consumed_write_units_percent = \
statistics.get_consumed_write_units_percent(table_name)
# Check if we should update write provisioning
if (consumed_write_units_percent == 0 and not
get_table_option(table_name, 'allow_scaling_down_writes_on_0_percent')):
logger.info(
'{0} - Scaling down writes is not done when usage is at 0%'.format(
table_name))
elif (consumed_write_units_percent >=
get_table_option(table_name, 'writes_upper_threshold')):
if get_table_option(table_name, 'increase_writes_unit') == 'percent':
updated_provisioning = calculators.increase_writes_in_percent(
table_name,
updated_write_units,
get_table_option(table_name, 'increase_writes_with'))
else:
updated_provisioning = calculators.increase_writes_in_units(
table_name,
updated_write_units,
get_table_option(table_name, 'increase_writes_with'))
update_needed = True
updated_write_units = updated_provisioning
elif (consumed_write_units_percent <=
get_table_option(table_name, 'writes_lower_threshold')):
if get_table_option(table_name, 'decrease_writes_unit') == 'percent':
updated_provisioning = calculators.decrease_writes_in_percent(
table_name,
updated_write_units,
get_table_option(table_name, 'decrease_writes_with'))
else:
updated_provisioning = calculators.decrease_writes_in_units(
table_name,
updated_write_units,
get_table_option(table_name, 'decrease_writes_with'))
update_needed = True
updated_write_units = updated_provisioning
return update_needed, int(updated_write_units)
def run(self):
""" Run the daemon
:type check_interval: int
:param check_interval: Delay in seconds between checks
"""
try:
while True:
execute()
except Exception as error:
logger.exception(error)
def run(self):
""" Run the daemon
:type check_interval: int
:param check_interval: Delay in seconds between checks
"""
try:
while True:
execute()
except Exception as error:
logger.exception(error)
def execute_table_in_loop(table_name, table_key, table_logger):
if not get_global_option('daemon') and get_global_option('run_once'):
execute_table(table_name, table_key)
else:
t = threading.currentThread()
while getattr(t, "do_run", True):
execute_table(table_name, table_key, table_logger)
logger.debug('Sleeping {0} seconds until next check'.format(
get_global_option('check_interval')))
time.sleep(get_global_option('check_interval'))
def get_provisioned_write_units(table_name):
""" Returns the number of provisioned write units for the table
:type table_name: str
:param table_name: Name of the DynamoDB table
:returns: int -- Number of write units
"""
table = dynamodb.get_table(table_name)
logger.debug('{0} - Provisioned write units: {1:d}'.format(
table_name, table.write_units))
return int(table.write_units)
def execute():
""" Ensure provisioning """
boto_server_error_retries = 3
# Ensure provisioning
for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()):
logger.debug("Table: " + table_name)
execute_in_thread(table_name, table_key)
# Sleep between the checks
if not get_global_option('run_once'):
logger.debug('Sleeping {0} seconds until next thread check'.format(60))
time.sleep(60)
def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
while not connected:
try:
if (configuration['global']['aws_access_key_id']
and configuration['global']['aws_secret_access_key']):
connection = dynamodb.connect_to_region(
configuration['global']['region'],
aws_access_key_id=configuration['global']['aws_access_key_id'],
aws_secret_access_key=\
configuration['global']['aws_secret_access_key'])
else:
connection = dynamodb.connect_to_region(
configuration['global']['region'])
connected = True
except Exception as err:
logger.error('Failed to connect to DynamoDB: {0}'.format(err))
if retries == 0:
logger.error(
'Please report an issue at: '
'https://github.com/sebdah/dynamic-dynamodb/issues')
raise
else:
logger.error('Retrying in 5 seconds')
retries -= 1
time.sleep(5)
logger.debug('Connected to DynamoDB')
return connection
def get_tables_and_gsis():
""" Get a set of tables and gsis and their configuration keys
:returns: set -- A set of tuples (table_name, table_conf_key)
"""
table_names = set()
configured_tables = get_configured_tables()
not_used_tables = set(configured_tables)
# Add regexp table names
for table_instance in list_tables():
for key_name in configured_tables:
try:
if re.match(key_name, table_instance.table_name):
logger.debug("Table {0} match with config key {1}".format(
table_instance.table_name, key_name))
table_names.add((table_instance.table_name, key_name))
not_used_tables.discard(key_name)
else:
logger.debug(
"Table {0} did not match with config key {1}".format(
table_instance.table_name, key_name))
except re.error:
logger.error(
'Invalid regular expression: "{0}"'.format(key_name))
sys.exit(1)
if not_used_tables:
logger.warning('No tables matching the following configured '
'tables found: {0}'.format(', '.join(not_used_tables)))
return sorted(table_names)
def list_tables():
""" Return list of DynamoDB tables available from AWS
:returns: list -- List of DynamoDB tables
"""
tables = []
try:
table_list = DYNAMODB_CONNECTION.list_tables()
while True:
for table_name in table_list[u'TableNames']:
tables.append(get_table(table_name))
if u'LastEvaluatedTableName' in table_list:
table_list = DYNAMODB_CONNECTION.list_tables(
table_list[u'LastEvaluatedTableName'])
else:
break
except DynamoDBResponseError as error:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'ResourceNotFoundException':
logger.error('No tables found')
elif dynamodb_error == 'AccessDeniedException':
logger.debug(
'Your AWS API keys lack access to listing tables. '
'That is an issue if you are trying to use regular '
'expressions in your table configuration.')
elif dynamodb_error == 'UnrecognizedClientException':
logger.error(
'Invalid security token. Are your AWS API keys correct?')
else:
logger.error(
(
'Unhandled exception: {0}: {1}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
dynamodb_error,
error.body['message']))
except JSONResponseError as error:
logger.error('Communication error: {0}'.format(error))
sys.exit(1)
return tables
def increase_writes_in_percent(current_provisioning, percent,
max_provisioned_writes,
consumed_write_units_percent, log_tag):
""" Increase the current_provisioning with percent %
:type current_provisioning: int
:param current_provisioning: The current provisioning
:type percent: int
:param percent: How many percent should we increase with
:returns: int -- New provisioning value
:type max_provisioned_writes: int
:param max_provisioned_writes: Configured max provisioned writes
:type consumed_write_units_percent: float
:param consumed_write_units_percent: Number of consumed write units
:type log_tag: str
:param log_tag: Prefix for the log
"""
current_provisioning = float(current_provisioning)
consumed_write_units_percent = float(consumed_write_units_percent)
percent = float(percent)
consumption_based_current_provisioning = \
int(math.ceil(current_provisioning*(consumed_write_units_percent/100)))
if consumption_based_current_provisioning > current_provisioning:
increase = int(
math.ceil(consumption_based_current_provisioning *
(percent / 100)))
updated_provisioning = consumption_based_current_provisioning + increase
else:
increase = int(math.ceil(current_provisioning *
(float(percent) / 100)))
updated_provisioning = current_provisioning + increase
if max_provisioned_writes > 0:
if updated_provisioning > max_provisioned_writes:
logger.info(
'{0} - Reached provisioned writes max limit: {1}'.format(
log_tag, max_provisioned_writes))
return max_provisioned_writes
logger.debug(
'{0} - Write provisioning will be increased to {1:d} units'.format(
log_tag, int(updated_provisioning)))
return updated_provisioning
def get_provisioned_table_write_units(table_name):
""" Returns the number of provisioned write units for the table
:type table_name: str
:param table_name: Name of the DynamoDB table
:returns: int -- Number of write units
"""
try:
desc = DYNAMODB_CONNECTION.describe_table(table_name)
except JSONResponseError:
raise
write_units = int(
desc[u'Table'][u'ProvisionedThroughput'][u'WriteCapacityUnits'])
logger.debug('{0} - Currently provisioned write units: {1:d}'.format(
table_name, write_units))
return write_units
def get_table(table_name):
""" Return the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:returns: boto.dynamodb.table.Table
"""
try:
table = Table(table_name, connection=DYNAMODB_CONNECTION)
except DynamoDBResponseError as error:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'ResourceNotFoundException':
logger.error('{0} - Table {1} not found'.format(
table_name, table_name))
raise
return table
def __publish(topic, message, subject=None):
""" Publish a message to a SNS topic
:type topic: str
:param topic: SNS topic to publish the message to
:type message: str
:param message: Message to send via SNS
:type subject: str
:param subject: Subject to use for e-mail notifications
:returns: None
"""
try:
SNS_CONNECTION.publish(topic=topic, message=message, subject=subject)
logger.info('Sent SNS notification to {0}'.format(topic))
except BotoServerError as error:
logger.error('Problem sending SNS notification: {0}'.format(
error.message))
return
def __calculate_always_decrease_rw_values(table_name, gsi_name, read_units,
provisioned_reads, write_units,
provisioned_writes):
""" Calculate values for always-decrease-rw-together
This will only return reads and writes decreases if both reads and writes
are lower than the current provisioning
:type table_name: str
:param table_name: Name of the DynamoDB table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type read_units: int
:param read_units: New read unit provisioning
:type provisioned_reads: int
:param provisioned_reads: Currently provisioned reads
:type write_units: int
:param write_units: New write unit provisioning
:type provisioned_writes: int
:param provisioned_writes: Currently provisioned writes
:returns: (int, int) -- (reads, writes)
"""
if read_units < provisioned_reads and write_units < provisioned_writes:
return (read_units, write_units)
if read_units < provisioned_reads:
logger.info(
'{0} - GSI: {1} - Reads could be decreased, '
'but we are waiting for writes to get lower than the threshold '
'before scaling down'.format(table_name, gsi_name))
read_units = provisioned_reads
elif write_units < provisioned_writes:
logger.info(
'{0} - GSI: {1} - Writes could be decreased, '
'but we are waiting for reads to get lower than the threshold '
'before scaling down'.format(table_name, gsi_name))
write_units = provisioned_writes
return (read_units, write_units)
