def update_table_provisioning(table_name, reads, writes):
""""""
table = get_table(table_name)
try:
table.update(
throughput={
'read': reads,
'write': writes
})
logger.info(
'{0} - Provisioning updated to {1} reads and {2} writes'.format(
table_name, reads, writes))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException',
'ValidationException',
'ResourceInUseException']
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(
table_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(table_name, exception, error.body['message']))
def get_tables_and_gsis():
""" Get a set of tables and gsis and their configuration keys
:returns: set -- A set of tuples (table_name, table_conf_key)
"""
table_names = set()
configured_tables = get_configured_tables()
not_used_tables = set(configured_tables)
# Add regexp table names
for table_instance in list_tables():
for key_name in configured_tables:
try:
if re.match(key_name, table_instance.table_name):
logger.debug("Table {0} match with config key {1}".format(table_instance.table_name, key_name))
table_names.add((table_instance.table_name, key_name))
not_used_tables.discard(key_name)
else:
logger.debug(
"Table {0} did not match with config key {1}".format(table_instance.table_name, key_name)
)
except re.error:
logger.error('Invalid regular expression: "{0}"'.format(key_name))
sys.exit(1)
if not_used_tables:
logger.warning(
"No tables matching the following configured " "tables found: {0}".format(", ".join(not_used_tables))
)
return sorted(table_names)
def get_tables_and_gsis():
""" Get a set of tables and gsis and their configuration keys
:returns: set -- A set of tuples (table_name, table_conf_key)
"""
table_names = set()
configured_tables = get_configured_tables()
not_used_tables = set(configured_tables)
# Add regexp table names
for table_instance in list_tables():
for key_name in configured_tables:
try:
if re.match(key_name, table_instance.table_name):
logger.debug("Table {0} match with config key {1}".format(
table_instance.table_name, key_name))
table_names.add((table_instance.table_name, key_name))
not_used_tables.discard(key_name)
else:
logger.debug(
"Table {0} did not match with config key {1}".format(
table_instance.table_name, key_name))
except re.error:
logger.error(
'Invalid regular expression: "{0}"'.format(key_name))
sys.exit(1)
if not_used_tables:
logger.warning('No tables matching the following configured '
'tables found: {0}'.format(', '.join(not_used_tables)))
return sorted(table_names)
def ensure_provisioning(table_name, key_name):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
"""
if get_global_option('circuit_breaker_url'):
if circuit_breaker.is_open():
logger.warning('Circuit breaker is OPEN!')
return None
read_update_needed, updated_read_units = __ensure_provisioning_reads(
table_name, key_name)
write_update_needed, updated_write_units = __ensure_provisioning_writes(
table_name, key_name)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
__update_throughput(
table_name, updated_read_units, updated_write_units, key_name)
else:
logger.info('{0} - No need to change provisioning'.format(table_name))
def ensure_provisioning(table_name, key_name):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
"""
if get_global_option('circuit_breaker_url'):
if __circuit_breaker_is_open():
logger.warning('Circuit breaker is OPEN!')
return None
read_update_needed, updated_read_units = __ensure_provisioning_reads(
table_name, key_name)
write_update_needed, updated_write_units = __ensure_provisioning_writes(
table_name, key_name)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
update_throughput(table_name, updated_read_units, updated_write_units, key_name)
else:
logger.info('{0} - No need to change provisioning'.format(table_name))
def ensure_provisioning(table_name, key_name, num_consec_read_checks,
num_consec_write_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option('circuit_breaker_url') or get_table_option(
key_name, 'circuit_breaker_url'):
if circuit_breaker.is_open(table_name, key_name):
logger.warning('Circuit breaker is OPEN!')
return (0, 0)
# Handle throughput alarm checks
__ensure_provisioning_alarm(table_name, key_name)
try:
read_update_needed, updated_read_units, num_consec_read_checks = \
__ensure_provisioning_reads(
table_name,
key_name,
num_consec_read_checks)
write_update_needed, updated_write_units, num_consec_write_checks = \
__ensure_provisioning_writes(
table_name,
key_name,
num_consec_write_checks)
if read_update_needed:
num_consec_read_checks = 0
if write_update_needed:
num_consec_write_checks = 0
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info('{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name, int(updated_read_units),
int(updated_write_units)))
__update_throughput(table_name, key_name, updated_read_units,
updated_write_units)
else:
logger.info(
'{0} - No need to change provisioning'.format(table_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
def ensure_provisioning(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks, num_consec_write_checks):
""" Ensure that provisioning is correct for Global Secondary Indexes
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option("circuit_breaker_url"):
if circuit_breaker.is_open():
logger.warning("Circuit breaker is OPEN!")
return (0, 0)
logger.info("{0} - Will ensure provisioning for global secondary index {1}".format(table_name, gsi_name))
# Handle throughput alarm checks
__ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key)
try:
read_update_needed, updated_read_units, num_consec_read_checks = __ensure_provisioning_reads(
table_name, table_key, gsi_name, gsi_key, num_consec_read_checks
)
write_update_needed, updated_write_units, num_consec_write_checks = __ensure_provisioning_writes(
table_name, table_key, gsi_name, gsi_key, num_consec_write_checks
)
if read_update_needed:
num_consec_read_checks = 0
if write_update_needed:
num_consec_write_checks = 0
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
"{0} - GSI: {1} - Changing provisioning to {2:d} "
"read units and {3:d} write units".format(
table_name, gsi_name, int(updated_read_units), int(updated_write_units)
)
)
__update_throughput(table_name, table_key, gsi_name, gsi_key, updated_read_units, updated_write_units)
else:
logger.info("{0} - GSI: {1} - No need to change provisioning".format(table_name, gsi_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
def ensure_provisioning(table_name, table_key, gsi_name, gsi_key):
""" Ensure that provisioning is correct for Global Secondary Indexes
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
"""
if get_global_option('circuit_breaker_url'):
if circuit_breaker.is_open():
logger.warning('Circuit breaker is OPEN!')
return None
logger.info(
'{0} - Will ensure provisioning for global secondary index {1}'.format(
table_name, gsi_name))
try:
read_update_needed, updated_read_units = __ensure_provisioning_reads(
table_name,
table_key,
gsi_name,
gsi_key)
write_update_needed, updated_write_units = __ensure_provisioning_writes(
table_name,
table_key,
gsi_name,
gsi_key)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - GSI: {1} - Changing provisioning to {2:d} '
'read units and {3:d} write units'.format(
table_name,
gsi_name,
int(updated_read_units),
int(updated_write_units)))
__update_throughput(
table_name,
table_key,
gsi_name,
gsi_key,
updated_read_units,
updated_write_units)
else:
logger.info(
'{0} - GSI: {1} - No need to change provisioning'.format(
table_name,
gsi_name))
except JSONResponseError:
raise
def ensure_provisioning(
table_name, key_name,
num_consec_read_checks,
num_consec_write_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option('circuit_breaker_url'):
if circuit_breaker.is_open():
logger.warning('Circuit breaker is OPEN!')
return (0, 0)
try:
read_update_needed, updated_read_units, num_consec_read_checks = \
__ensure_provisioning_reads(
table_name,
key_name,
num_consec_read_checks)
write_update_needed, updated_write_units, num_consec_write_checks = \
__ensure_provisioning_writes(
table_name,
key_name,
num_consec_write_checks)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
__update_throughput(
table_name,
key_name,
updated_read_units,
updated_write_units)
else:
logger.info('{0} - No need to change provisioning'.format(
table_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units,
write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
current_wu = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
except JSONResponseError:
raise
# Check table status
try:
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
except JSONResponseError:
raise
logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
table_name, gsi_name, gsi_status))
if gsi_status != 'ACTIVE':
logger.warning(
'{0} - GSI: {1} - Not performing throughput changes when GSI '
'status is {2}'.format(table_name, gsi_name, gsi_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name, gsi_name, read_units, current_ru, write_units,
current_wu)
if read_units == current_ru and write_units == current_wu:
logger.info('{0} - GSI: {1} - No changes to perform'.format(
table_name, gsi_name))
return
dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key,
int(read_units), int(write_units))
def __update_throughput(table_name, key_name, read_units, write_units):
""" Update throughput on the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_table_read_units(table_name)
current_wu = dynamodb.get_provisioned_table_write_units(table_name)
except JSONResponseError:
raise
# Check table status
try:
table_status = dynamodb.get_table_status(table_name)
except JSONResponseError:
raise
logger.debug('{0} - Table status is {1}'.format(table_name, table_status))
if table_status != 'ACTIVE':
logger.warning(
'{0} - Not performing throughput changes when table '
'is {1}'.format(table_name, table_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_table_option(key_name, 'always_decrease_rw_together'):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name,
read_units,
current_ru,
write_units,
current_wu)
if read_units == current_ru and write_units == current_wu:
logger.info('{0} - No changes to perform'.format(table_name))
return
dynamodb.update_table_provisioning(
table_name,
key_name,
int(read_units),
int(write_units))
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name)
current_wu = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name)
except JSONResponseError:
raise
# Check table status
try:
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
except JSONResponseError:
raise
logger.debug("{0} - GSI: {1} - GSI status is {2}".format(table_name, gsi_name, gsi_status))
if gsi_status != "ACTIVE":
logger.warning(
"{0} - GSI: {1} - Not performing throughput changes when GSI "
"status is {2}".format(table_name, gsi_name, gsi_status)
)
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, "always_decrease_rw_together"):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name, gsi_name, read_units, current_ru, write_units, current_wu
)
if read_units == current_ru and write_units == current_wu:
logger.info("{0} - GSI: {1} - No changes to perform".format(table_name, gsi_name))
return
dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def get_tables_and_gsis():
""" Get a set of tables and gsis and their configuration keys
:returns: set -- A set of tuples (table_name, table_conf_key)
"""
table_names = set()
configured_tables = get_configured_tables()
not_used_tables = set(configured_tables)
# Add regexp table names
for table_instance in list_tables():
for key_name in configured_tables:
try:
if re.match(key_name, table_instance.table_name):
logger.debug("Table {0} match with config key {1}".format(
table_instance.table_name, key_name))
# Notify users about regexps that match multiple tables
if table_instance.table_name in [x[0] for x in table_names]:
logger.warning(
'Table {0} matches more than one regexp in config, '
'skipping this match: "{1}"'.format(
table_instance.table_name, key_name))
else:
table_names.add(
(
table_instance.table_name,
key_name
))
not_used_tables.discard(key_name)
else:
logger.debug(
"Table {0} did not match with config key {1}".format(
table_instance.table_name, key_name))
except re.error:
logger.error('Invalid regular expression: "{0}"'.format(
key_name))
sys.exit(1)
if not_used_tables:
logger.warning(
'No tables matching the following configured '
'tables found: {0}'.format(', '.join(not_used_tables)))
return sorted(table_names)
def update_gsi_provisioning(table_name, gsi_name, reads, writes):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
"""
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits": reads,
"WriteCapacityUnits": writes
}
}
}
])
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name, gsi_name, exception, error.body['message']))
def update_throughput(table_name, read_units, write_units, key_name):
""" Update throughput on the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
:type key_name: str
:param key_name: Configuration option key name
"""
try:
table = dynamodb.get_table(table_name)
except DynamoDBResponseError:
# Return if the table does not exist
return None
# Check that we are in the right time frame
if get_table_option(key_name, 'maintenance_windows'):
if (not __is_maintenance_window(table_name, get_table_option(
key_name, 'maintenance_windows'))):
logger.warning(
'{0} - Current time is outside maintenance window'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
# Check table status
if table.status != 'ACTIVE':
logger.warning(
'{0} - Not performing throughput changes when table '
'is in {1} state'.format(table_name, table.status))
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_table_option(key_name, 'always_decrease_rw_together'):
if ((read_units < table.read_units) or
(table.read_units == get_table_option(
key_name, 'min_provisioned_reads'))):
if ((write_units < table.write_units) or
(table.write_units == get_table_option(
key_name, 'min_provisioned_writes'))):
logger.info(
'{0} - Both reads and writes will be decreased'.format(
table_name))
elif read_units < table.read_units:
logger.info(
'{0} - Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(table_name))
read_units = table.read_units
elif write_units < table.write_units:
logger.info(
'{0} - Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(table_name))
write_units = table.write_units
if read_units == table.read_units and write_units == table.write_units:
logger.debug('{0} - No need to update provisioning')
return
if not get_global_option('dry_run'):
try:
table.update_throughput(int(read_units), int(write_units))
logger.info('Provisioning updated')
except DynamoDBResponseError as error:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'LimitExceededException':
logger.warning(
'{0} - {1}'.format(table_name, error.body['message']))
if int(read_units) > table.read_units:
logger.info('{0} - Scaling up reads to {1:d}'.format(
table_name,
int(read_units)))
update_throughput(
table_name,
int(read_units),
int(table.write_units),
key_name)
elif int(write_units) > table.write_units:
logger.info('{0} - Scaling up writes to {1:d}'.format(
table_name,
int(write_units)))
update_throughput(
table_name,
int(table.read_units),
int(write_units),
key_name)
elif dynamodb_error == 'ValidationException':
logger.warning('{0} - ValidationException: {1}'.format(
table_name,
error.body['message']))
elif dynamodb_error == 'ResourceInUseException':
logger.warning('{0} - ResourceInUseException: {1}'.format(
table_name,
error.body['message']))
elif dynamodb_error == 'AccessDeniedException':
logger.warning('{0} - AccessDeniedException: {1}'.format(
table_name,
error.body['message']))
else:
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name,
dynamodb_error,
error.body['message']))
def is_open(table_name=None, table_key=None, gsi_name=None, gsi_key=None):
""" Checks whether the circuit breaker is open
:param table_name: Name of the table being checked
:param table_key: Configuration key for table
:param gsi_name: Name of the GSI being checked
:param gsi_key: Configuration key for the GSI
:returns: bool -- True if the circuit is open
"""
logger.debug('Checking circuit breaker status')
# Parse the URL to make sure it is OK
pattern = re.compile(
r'^(?P<scheme>http(s)?://)'
r'((?P<username>.+):(?P<password>.+)@){0,1}'
r'(?P<url>.*)$'
)
url = timeout = None
if gsi_name:
url = get_gsi_option(table_key, gsi_key, 'circuit_breaker_url')
timeout = get_gsi_option(table_key, gsi_key, 'circuit_breaker_timeout')
elif table_name:
url = get_table_option(table_key, 'circuit_breaker_url')
timeout = get_table_option(table_key, 'circuit_breaker_timeout')
if not url:
url = get_global_option('circuit_breaker_url')
timeout = get_global_option('circuit_breaker_timeout')
match = pattern.match(url)
if not match:
logger.error('Malformatted URL: {0}'.format(url))
sys.exit(1)
use_basic_auth = False
if match.group('username') and match.group('password'):
use_basic_auth = True
# Make the actual URL to call
auth = ()
if use_basic_auth:
url = '{scheme}{url}'.format(
scheme=match.group('scheme'),
url=match.group('url'))
auth = (match.group('username'), match.group('password'))
headers = {}
if table_name:
headers["x-table-name"] = table_name
if gsi_name:
headers["x-gsi-name"] = gsi_name
# Make the actual request
try:
response = requests.get(
url,
auth=auth,
timeout=timeout / 1000.00,
headers=headers)
if int(response.status_code) == 200:
logger.info('Circuit breaker is closed')
return False
else:
logger.warning(
'Circuit breaker returned with status code {0:d}'.format(
response.status_code))
except requests.exceptions.SSLError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.Timeout as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.ConnectionError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.HTTPError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.TooManyRedirects as error:
logger.warning('Circuit breaker: {0}'.format(error))
except Exception as error:
logger.error('Unhandled exception: {0}'.format(error))
logger.error(
'Please file a bug at '
'https://github.com/sebdah/dynamic-dynamodb/issues')
return True
def update_gsi_provisioning(table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
# Check that we are in the right time frame
m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows')
if m_windows:
if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
logger.warning(
'{0} - GSI: {1} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(
table_name, gsi_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info('{0} - GSI: {1} - '
'No need to scale up reads nor writes'.format(
table_name, gsi_name))
return
else:
logger.info('{0} - GSI: {1} - '
'Current time is within maintenance window'.format(
table_name, gsi_name))
logger.info('{0} - GSI: {1} - '
'Updating provisioning to {2} reads and {3} writes'.format(
table_name, gsi_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
DYNAMODB_CONNECTION.update_table(table_name=table_name,
global_secondary_index_updates=[{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits":
reads,
"WriteCapacityUnits":
writes
}
}
}])
message = ('{0} - GSI: {1} - Provisioning updated to '
'{2} reads and {3} writes').format(table_name, gsi_name,
reads, writes)
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < current_writes:
sns_message_types.append('scale-up')
sns.publish_gsi_notification(
table_key,
gsi_key,
message,
sns_message_types,
subject='Updated provisioning for GSI {0}'.format(gsi_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
('{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues').format(
table_name, gsi_name, exception, error.body['message']))
if (not retry_with_only_increase
and exception == 'LimitExceededException'):
logger.info('{0} - GSI: {1} - Will retry to update provisioning '
'with only increases'.format(table_name, gsi_name))
update_gsi_provisioning(table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=True)
def update_table_provisioning(table_name,
key_name,
reads,
writes,
retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Configuration option key name
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
# Check that we are in the right time frame
maintenance_windows = get_table_option(key_name, 'maintenance_windows')
if maintenance_windows:
if not __is_table_maintenance_window(table_name, maintenance_windows):
logger.warning(
'{0} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(table_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
logger.info(
'{0} - Updating provisioning to {1} reads and {2} writes'.format(
table_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
table.update(throughput={'read': reads, 'write': writes})
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < current_writes:
sns_message_types.append('scale-up')
message = (
'{0} - Provisioning updated to {1} reads and {2} writes').format(
table_name, reads, writes)
sns.publish_table_notification(
key_name,
message,
sns_message_types,
subject='Updated provisioning for table {0}'.format(table_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException', 'ValidationException',
'ResourceInUseException'
]
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(table_name, exception,
error.body['message']))
else:
logger.error(
('{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues').format(
table_name, exception, error.body['message']))
if (not retry_with_only_increase
and exception == 'LimitExceededException'):
logger.info('{0} - Will retry to update provisioning '
'with only increases'.format(table_name))
update_table_provisioning(table_name,
key_name,
reads,
writes,
retry_with_only_increase=True)
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name))
current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info("{0} - GSI: {1} - No need to scale up reads nor writes".format(table_name, gsi_name))
return
logger.info(
"{0} - GSI: {1} - Retrying to update provisioning, excluding any decreases. "
"Setting new reads to {2} and new writes to {3}".format(table_name, gsi_name, reads, writes)
)
# Check that we are in the right time frame
m_windows = get_gsi_option(table_key, gsi_key, "maintenance_windows")
if m_windows:
if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
logger.warning(
"{0} - GSI: {1} - We are outside a maintenace window. "
"Will only perform up scaling activites".format(table_name, gsi_name)
)
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info("{0} - GSI: {1} - " "No need to scale up reads nor writes".format(table_name, gsi_name))
return
else:
logger.info("{0} - GSI: {1} - " "Current time is within maintenance window".format(table_name, gsi_name))
logger.info(
"{0} - GSI: {1} - "
"Updating provisioning to {2} reads and {3} writes".format(table_name, gsi_name, reads, writes)
)
# Return if dry-run
if get_global_option("dry_run"):
return
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {"ReadCapacityUnits": reads, "WriteCapacityUnits": writes},
}
}
],
)
message = ("{0} - GSI: {1} - Provisioning updated to " "{2} reads and {3} writes").format(
table_name, gsi_name, reads, writes
)
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append("scale-down")
if current_reads < reads or current_writes < current_writes:
sns_message_types.append("scale-up")
sns.publish_gsi_notification(
table_key, gsi_key, message, sns_message_types, subject="Updated provisioning for GSI {0}".format(gsi_name)
)
except JSONResponseError as error:
exception = error.body["__type"].split("#")[1]
know_exceptions = ["LimitExceededException"]
if exception in know_exceptions:
logger.warning("{0} - GSI: {1} - {2}: {3}".format(table_name, gsi_name, exception, error.body["message"]))
else:
logger.error(
(
"{0} - GSI: {1} - Unhandled exception: {2}: {3}. "
"Please file a bug report at "
"https://github.com/sebdah/dynamic-dynamodb/issues"
).format(table_name, gsi_name, exception, error.body["message"])
)
if not retry_with_only_increase and exception == "LimitExceededException":
logger.info(
"{0} - GSI: {1} - Will retry to update provisioning " "with only increases".format(table_name, gsi_name)
)
update_gsi_provisioning(
table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True
)
def __update_throughput(table_name, read_units, write_units, key_name):
""" Update throughput on the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
:type key_name: str
:param key_name: Configuration option key name
"""
provisioned_reads = table_stats.get_provisioned_read_units(table_name)
provisioned_writes = table_stats.get_provisioned_write_units(table_name)
# Check that we are in the right time frame
if get_table_option(key_name, 'maintenance_windows'):
if (not __is_maintenance_window(table_name, get_table_option(
key_name, 'maintenance_windows'))):
logger.warning(
'{0} - Current time is outside maintenance window'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
# Check table status
table_status = dynamodb.get_table_status(table_name)
logger.debug('{0} - Table status is {1}'.format(table_name, table_status))
if table_status != 'ACTIVE':
logger.warning(
'{0} - Not performing throughput changes when table '
'is {1}'.format(table_name, table_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_table_option(key_name, 'always_decrease_rw_together'):
if read_units < provisioned_reads and write_units < provisioned_writes:
logger.debug(
'{0} - Both reads and writes will be decreased'.format(
table_name))
elif read_units < provisioned_reads:
logger.info(
'{0} - Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(table_name))
return
elif write_units < provisioned_writes:
logger.info(
'{0} - Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(table_name))
return
if not get_global_option('dry_run'):
dynamodb.update_table_provisioning(
table_name,
int(read_units),
int(write_units))
def is_open(table_name=None, table_key=None, gsi_name=None, gsi_key=None):
""" Checks whether the circuit breaker is open
:param table_name: Name of the table being checked
:param table_key: Configuration key for table
:param gsi_name: Name of the GSI being checked
:param gsi_key: Configuration key for the GSI
:returns: bool -- True if the circuit is open
"""
logger.debug('Checking circuit breaker status')
# Parse the URL to make sure it is OK
pattern = re.compile(r'^(?P<scheme>http(s)?://)'
r'((?P<username>.+):(?P<password>.+)@){0,1}'
r'(?P<url>.*)$')
url = timeout = None
if gsi_name:
url = get_gsi_option(table_key, gsi_key, 'circuit_breaker_url')
timeout = get_gsi_option(table_key, gsi_key, 'circuit_breaker_timeout')
elif table_name:
url = get_table_option(table_key, 'circuit_breaker_url')
timeout = get_table_option(table_key, 'circuit_breaker_timeout')
if not url:
url = get_global_option('circuit_breaker_url')
timeout = get_global_option('circuit_breaker_timeout')
match = pattern.match(url)
if not match:
logger.error('Malformatted URL: {0}'.format(url))
sys.exit(1)
use_basic_auth = False
if match.group('username') and match.group('password'):
use_basic_auth = True
# Make the actual URL to call
auth = ()
if use_basic_auth:
url = '{scheme}{url}'.format(scheme=match.group('scheme'),
url=match.group('url'))
auth = (match.group('username'), match.group('password'))
headers = {}
if table_name:
headers["x-table-name"] = table_name
if gsi_name:
headers["x-gsi-name"] = gsi_name
# Make the actual request
try:
response = requests.get(url,
auth=auth,
timeout=timeout / 1000.00,
headers=headers)
if int(response.status_code) == 200:
logger.info('Circuit breaker is closed')
return False
else:
logger.warning(
'Circuit breaker returned with status code {0:d}'.format(
response.status_code))
except requests.exceptions.SSLError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.Timeout as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.ConnectionError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.HTTPError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.TooManyRedirects as error:
logger.warning('Circuit breaker: {0}'.format(error))
except Exception as error:
logger.error('Unhandled exception: {0}'.format(error))
logger.error('Please file a bug at '
'https://github.com/sebdah/dynamic-dynamodb/issues')
return True
def __circuit_breaker_is_open():
""" Checks wether the circuit breaker is open
:returns: bool -- True if the circuit is open
"""
logger.debug('Checking circuit breaker status')
# Parse the URL to make sure it is OK
pattern = re.compile(
r'^(?P<scheme>http(s)?://)'
r'((?P<username>.+):(?P<password>.+)@){0,1}'
r'(?P<url>.*)$'
)
match = pattern.match(get_global_option('circuit_breaker_url'))
if not match:
logger.error('Malformatted URL: {0}'.format(
get_global_option('circuit_breaker_url')))
sys.exit(1)
use_basic_auth = False
if match.group('username') and match.group('password'):
use_basic_auth = True
# Make the actual URL to call
if use_basic_auth:
url = '{scheme}{url}'.format(
scheme=match.group('scheme'),
url=match.group('url'))
auth = (match.group('username'), match.group('password'))
else:
url = get_global_option('circuit_breaker_url')
auth = ()
# Make the actual request
try:
response = requests.get(
url,
auth=auth,
timeout=get_global_option('circuit_breaker_timeout') / 1000.00)
if int(response.status_code) == 200:
logger.info('Circuit breaker is closed')
return False
else:
logger.warning(
'Circuit breaker returned with status code {0:d}'.format(
response.status_code))
except requests.exceptions.SSLError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.Timeout as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.ConnectionError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.HTTPError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.TooManyRedirects as error:
logger.warning('Circuit breaker: {0}'.format(error))
except Exception as error:
logger.error('Unhandled exception: {0}'.format(error))
logger.error(
'Please file a bug at '
'https://github.com/sebdah/dynamic-dynamodb/issues')
return True
def __update_throughput(
table_name, table_key, gsi_name, gsi_key, read_units, write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
try:
current_ru = dynamodb.get_provisioned_gsi_read_units(
table_name, gsi_name)
current_wu = dynamodb.get_provisioned_gsi_write_units(
table_name, gsi_name)
except JSONResponseError:
raise
# Check that we are in the right time frame
if get_gsi_option(table_key, gsi_key, 'maintenance_windows'):
if (not __is_maintenance_window(table_name, gsi_name, get_gsi_option(
table_key, gsi_key, 'maintenance_windows'))):
logger.warning(
'{0} - GSI: {1} - '
'Current time is outside maintenance window'.format(
table_name,
gsi_name))
return
else:
logger.info(
'{0} - GSI: {1} - '
'Current time is within maintenance window'.format(
table_name,
gsi_name))
# Check table status
try:
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
except JSONResponseError:
raise
logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
table_name, gsi_name, gsi_status))
if gsi_status != 'ACTIVE':
logger.warning(
'{0} - GSI: {1} - Not performing throughput changes when GSI '
'status is {2}'.format(table_name, gsi_name, gsi_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
read_units, write_units = __calculate_always_decrease_rw_values(
table_name,
gsi_name,
read_units,
current_ru,
write_units,
current_wu)
if read_units == current_ru and write_units == current_wu:
logger.info('{0} - GSI: {1} - No changes to perform'.format(
table_name, gsi_name))
return
if not get_global_option('dry_run'):
dynamodb.update_gsi_provisioning(
table_name,
table_key,
gsi_name,
gsi_key,
int(read_units),
int(write_units))
logger.info(
'{0} - GSI: {1} - '
'Provisioning updated to {2} reads and {3} writes'.format(
table_name,
gsi_name,
read_units,
write_units))
def __update_throughput(
table_name, table_key, gsi_name, gsi_key, read_units, write_units):
""" Update throughput on the GSI
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: Configuration option key name
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
"""
current_ru = gsi_stats.get_provisioned_read_units(
table_name, gsi_name)
current_wu = gsi_stats.get_provisioned_write_units(
table_name, gsi_name)
# Check that we are in the right time frame
if get_gsi_option(table_key, gsi_key, 'maintenance_windows'):
if (not __is_maintenance_window(table_name, gsi_name, get_gsi_option(
table_key, gsi_key, 'maintenance_windows'))):
logger.warning(
'{0} - GSI: {1} - '
'Current time is outside maintenance window'.format(
table_name,
gsi_name))
return
else:
logger.info(
'{0} - GSI: {1} - '
'Current time is within maintenance window'.format(
table_name,
gsi_name))
# Check table status
gsi_status = dynamodb.get_gsi_status(table_name, gsi_name)
logger.debug('{0} - GSI: {1} - GSI status is {2}'.format(
table_name, gsi_name, gsi_status))
if gsi_status != 'ACTIVE':
logger.warning(
'{0} - GSI: {1} - Not performing throughput changes when GSI '
'status is {2}'.format(table_name, gsi_name, gsi_status))
return
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'):
if read_units < current_ru and write_units < current_wu:
logger.debug(
'{0} - GSI: {1} - '
'Both reads and writes will be decreased'.format(
table_name,
gsi_name))
elif read_units < current_ru:
logger.info(
'{0} - GSI: {1} - '
'Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(
table_name, gsi_name))
return
elif write_units < current_wu:
logger.info(
'{0} - GSI: {1} - '
'Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(
table_name, gsi_name))
return
if not get_global_option('dry_run'):
dynamodb.update_gsi_provisioning(
table_name,
gsi_name,
int(read_units),
int(write_units))
logger.info(
'{0} - GSI: {1} - '
'Provisioning updated to {2} reads and {3} writes'.format(
table_name,
gsi_name,
read_units,
write_units))
def main():
""" Main function called from dynamic-dynamodb """
while True:
table_names = set()
configured_tables = config['tables'].keys()
not_used_tables = configured_tables
# Add regexp table names
for table_instance in dynamodb.list_tables():
for key_name in configured_tables:
if re.match(key_name, table_instance.table_name):
logger.debug("Table {0} match with config key {1}".format(
table_instance.table_name, key_name))
table_names.add(
(
table_instance.table_name,
key_name
))
not_used_tables.remove(key_name)
if not_used_tables:
logger.warning(
'No tables matching the following configured '
'tables found: {0}'.format(
', '.join(not_used_tables)))
table_names = sorted(table_names)
if config['global']['daemon']:
pid_file = '/tmp/dynamic-dynamodb.{0}.pid'.format(
config['global']['instance'])
daemon = DynamicDynamoDBDaemon(pid_file, tables=table_names)
if config['global']['daemon'] == 'start':
daemon.start(
check_interval=config['global']['check_interval'])
elif config['global']['daemon'] == 'stop':
daemon.stop()
sys.exit(0)
elif config['global']['daemon'] == 'restart':
daemon.restart()
elif config['global']['daemon'] in ['foreground', 'fg']:
daemon.run(
check_interval=config['global']['check_interval'])
else:
print 'Valid options for --daemon are start, stop and restart'
sys.exit(1)
else:
# Ensure provisioning
for table_name, table_key in table_names:
table.ensure_provisioning(table_name, table_key)
gsi_names = set()
# Add regexp table names
if 'gsis' in config['tables'][table_key]:
for gst_instance in dynamodb.table_gsis(table_name):
gsi_name = gst_instance[u'IndexName']
gsi_keys = config['tables'][table_key]['gsis'].keys()
for gsi_key in gsi_keys:
if re.match(gsi_key, gsi_name):
logger.debug(
'Table {0} GSI {1} match with '
'GSI config key {2}'.format(
table_name, gsi_name, gsi_key))
gsi_names.add(
(
gsi_name,
gsi_key
))
gsi_names = sorted(gsi_names)
for gsi_name, gsi_key in gsi_names:
gsi.ensure_provisioning(
table_name,
table_key,
gsi_name,
gsi_key)
# Sleep between the checks
logger.debug('Sleeping {0} seconds until next check'.format(
config['global']['check_interval']))
time.sleep(config['global']['check_interval'])
def update_table_provisioning(
table_name, reads, writes, retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
if retry_with_only_increase:
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
try:
table.update(
throughput={
'read': reads,
'write': writes
})
logger.info(
'{0} - Provisioning updated to {1} reads and {2} writes'.format(
table_name, reads, writes))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException',
'ValidationException',
'ResourceInUseException']
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(
table_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(table_name, exception, error.body['message']))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - Will retry to update provisioning '
'with only increases'.format(table_name))
update_table_provisioning(
table_name,
reads,
writes,
retry_with_only_increase=True)
def ensure_provisioning(
table_name, key_name,
num_consec_read_checks,
num_consec_write_checks):
""" Ensure that provisioning is correct
:type table_name: str
:param table_name: Name of the DynamoDB table
:type key_name: str
:param key_name: Configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option('circuit_breaker_url') or get_table_option(
key_name, 'circuit_breaker_url'):
if circuit_breaker.is_open(table_name, key_name):
logger.warning('Circuit breaker is OPEN!')
return (0, 0)
# Handle throughput alarm checks
__ensure_provisioning_alarm(table_name, key_name)
ts = TimeSeriesTable(get_table_option('time_series_tables', 'time_series_tables'),
get_table_option('time_series_tables_no_scale_period_in_seconds', 'time_series_tables_no_scale_period_in_seconds'))
if ts.is_in_future(table_name):
logger.info('Time series table ' + table_name + " is in the future, skipping provisioning")
return (0, 0)
try:
read_update_needed, updated_read_units, num_consec_read_checks = \
__ensure_provisioning_reads(
table_name,
key_name,
num_consec_read_checks)
write_update_needed, updated_write_units, num_consec_write_checks = \
__ensure_provisioning_writes(
table_name,
key_name,
num_consec_write_checks)
if read_update_needed:
num_consec_read_checks = 0
if write_update_needed:
num_consec_write_checks = 0
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info(
'{0} - Changing provisioning to {1:d} '
'read units and {2:d} write units'.format(
table_name,
int(updated_read_units),
int(updated_write_units)))
__update_throughput(
table_name,
key_name,
updated_read_units,
updated_write_units)
else:
logger.info('{0} - No need to change provisioning'.format(
table_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
def __circuit_breaker_is_open():
""" Checks wether the circuit breaker is open
:returns: bool -- True if the circuit is open
"""
logger.debug('Checking circuit breaker status')
# Parse the URL to make sure it is OK
pattern = re.compile(
r'^(?P<scheme>http(s)?://)'
r'((?P<username>.+):(?P<password>.+)@){0,1}'
r'(?P<url>.*)$'
)
match = pattern.match(get_global_option('circuit_breaker_url'))
if not match:
logger.error('Malformatted URL: {0}'.format(
get_global_option('circuit_breaker_url')))
sys.exit(1)
use_basic_auth = False
if match.group('username') and match.group('password'):
use_basic_auth = True
# Make the actual URL to call
if use_basic_auth:
url = '{scheme}{url}'.format(
scheme=match.group('scheme'),
url=match.group('url'))
auth = (match.group('username'), match.group('password'))
else:
url = get_global_option('circuit_breaker_url')
auth = ()
# Make the actual request
try:
response = requests.get(
url,
auth=auth,
timeout=get_global_option('circuit_breaker_timeout') / 1000.00)
if int(response.status_code) == 200:
logger.info('Circuit breaker is closed')
return False
else:
logger.warning(
'Circuit breaker returned with status code {0:d}'.format(
response.status_code))
except requests.exceptions.SSLError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.Timeout as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.ConnectionError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.HTTPError as error:
logger.warning('Circuit breaker: {0}'.format(error))
except requests.exceptions.TooManyRedirects as error:
logger.warning('Circuit breaker: {0}'.format(error))
except Exception as error:
logger.error('Unhandled exception: {0}'.format(error))
logger.error(
'Please file a bug at '
'https://github.com/sebdah/dynamic-dynamodb/issues')
return True
def update_table_provisioning(
table_name, key_name, reads, writes, retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Configuration option key name
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
# Make sure we aren't scaling down if we turned off downscaling
if (not get_table_option(key_name, 'enable_reads_down_scaling') or
not get_table_option(key_name, 'enable_writes_down_scaling')):
if (not get_table_option(key_name, 'enable_reads_down_scaling') and
current_reads > reads):
reads = current_reads
if (not get_table_option(key_name, 'enable_writes_down_scaling') and
current_writes > writes):
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
# Check that we are in the right time frame
maintenance_windows = get_table_option(key_name, 'maintenance_windows')
if maintenance_windows:
if not __is_table_maintenance_window(table_name, maintenance_windows):
logger.warning(
'{0} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(table_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
logger.info(
'{0} - Updating provisioning to {1} reads and {2} writes'.format(
table_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
table.update(
throughput={
'read': reads,
'write': writes
})
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < writes:
sns_message_types.append('scale-up')
message = []
if current_reads > reads:
message.append('{0} - Reads: DOWN from {1} to {2}\n'.format(
table_name, current_reads, reads))
elif current_reads < reads:
message.append('{0} - Reads: UP from {1} to {2}\n'.format(
table_name, current_reads, reads))
if current_writes > writes:
message.append('{0} - Writes: DOWN from {1} to {2}\n'.format(
table_name, current_writes, writes))
elif current_writes < writes:
message.append('{0} - Writes: UP from {1} to {2}\n'.format(
table_name, current_writes, writes))
sns.publish_table_notification(
key_name,
''.join(message),
sns_message_types,
subject='Updated provisioning for table {0}'.format(table_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = [
'LimitExceededException',
'ValidationException',
'ResourceInUseException']
if exception in know_exceptions:
logger.warning('{0} - {1}: {2}'.format(
table_name, exception, error.body['message']))
else:
if 'message' in error.body:
msg = error.body['message']
else:
msg = error
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(table_name, exception, msg))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - Will retry to update provisioning '
'with only increases'.format(table_name))
update_table_provisioning(
table_name,
key_name,
reads,
writes,
retry_with_only_increase=True)
def update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=False):
""" Update provisioning for a given table
:type table_name: str
:param table_name: Name of the table
:type key_name: str
:param key_name: Configuration option key name
:type reads: int
:param reads: New number of provisioned read units
:type writes: int
:param writes: New number of provisioned write units
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
table = get_table(table_name)
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info("{0} - No need to scale up reads nor writes".format(table_name))
return
logger.info(
"{0} - Retrying to update provisioning, excluding any decreases. "
"Setting new reads to {1} and new writes to {2}".format(table_name, reads, writes)
)
# Check that we are in the right time frame
maintenance_windows = get_table_option(key_name, "maintenance_windows")
if maintenance_windows:
if not __is_table_maintenance_window(table_name, maintenance_windows):
logger.warning(
"{0} - We are outside a maintenace window. " "Will only perform up scaling activites".format(table_name)
)
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info("{0} - No need to scale up reads nor writes".format(table_name))
return
else:
logger.info("{0} - Current time is within maintenance window".format(table_name))
logger.info("{0} - Updating provisioning to {1} reads and {2} writes".format(table_name, reads, writes))
# Return if dry-run
if get_global_option("dry_run"):
return
try:
table.update(throughput={"read": reads, "write": writes})
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append("scale-down")
if current_reads < reads or current_writes < current_writes:
sns_message_types.append("scale-up")
message = ("{0} - Provisioning updated to {1} reads and {2} writes").format(table_name, reads, writes)
sns.publish_table_notification(
key_name, message, sns_message_types, subject="Updated provisioning for table {0}".format(table_name)
)
except JSONResponseError as error:
exception = error.body["__type"].split("#")[1]
know_exceptions = ["LimitExceededException", "ValidationException", "ResourceInUseException"]
if exception in know_exceptions:
logger.warning("{0} - {1}: {2}".format(table_name, exception, error.body["message"]))
else:
logger.error(
(
"{0} - Unhandled exception: {1}: {2}. "
"Please file a bug report at "
"https://github.com/sebdah/dynamic-dynamodb/issues"
).format(table_name, exception, error.body["message"])
)
if not retry_with_only_increase and exception == "LimitExceededException":
logger.info("{0} - Will retry to update provisioning " "with only increases".format(table_name))
update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=True)
def update_gsi_provisioning(
table_name, table_key, gsi_name, gsi_key,
reads, writes, retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name))
current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name))
# Make sure we aren't scaling down if we turned off downscaling
if (not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling') or
not get_gsi_option(
table_key, gsi_key, 'enable_writes_down_scaling')):
if (not get_gsi_option(
table_key, gsi_key, 'enable_reads_down_scaling') and
current_reads > reads):
reads = current_reads
if (not get_gsi_option(
table_key, gsi_key, 'enable_writes_down_scaling') and
current_writes > writes):
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - No need to scale up reads nor writes'.format(
table_name))
return
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale at all
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - GSI: {1} - No need to scale up reads nor writes'.format(
table_name, gsi_name))
return
logger.info(
'{0} - GSI: {1} - Retrying to update provisioning, '
'excluding any decreases. '
'Setting new reads to {2} and new writes to {3}'.format(
table_name, gsi_name, reads, writes))
# Check that we are in the right time frame
m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows')
if m_windows:
if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows):
logger.warning(
'{0} - GSI: {1} - We are outside a maintenace window. '
'Will only perform up scaling activites'.format(
table_name,
gsi_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
# Return if we do not need to scale up
if reads == current_reads and writes == current_writes:
logger.info(
'{0} - GSI: {1} - '
'No need to scale up reads nor writes'.format(
table_name,
gsi_name))
return
else:
logger.info(
'{0} - GSI: {1} - '
'Current time is within maintenance window'.format(
table_name,
gsi_name))
logger.info(
'{0} - GSI: {1} - '
'Updating provisioning to {2} reads and {3} writes'.format(
table_name, gsi_name, reads, writes))
# Return if dry-run
if get_global_option('dry_run'):
return
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits": reads,
"WriteCapacityUnits": writes
}
}
}
])
message = []
if current_reads > reads:
message.append(
'{0} - GSI: {1} - Reads: DOWN from {2} to {3}\n'.format(
table_name, gsi_name, current_reads, reads))
elif current_reads < reads:
message.append(
'{0} - GSI: {1} - Reads: UP from {2} to {3}\n'.format(
table_name, gsi_name, current_reads, reads))
if current_writes > writes:
message.append(
'{0} - GSI: {1} - Writes: DOWN from {2} to {3}\n'.format(
table_name, gsi_name, current_writes, writes))
elif current_writes < writes:
message.append(
'{0} - GSI: {1} - Writes: UP from {2} to {3}\n'.format(
table_name, gsi_name, current_writes, writes))
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < writes:
sns_message_types.append('scale-up')
sns.publish_gsi_notification(
table_key,
gsi_key,
''.join(message),
sns_message_types,
subject='Updated provisioning for GSI {0}'.format(gsi_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name, gsi_name, exception, error.body['message']))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - GSI: {1} - Will retry to update provisioning '
'with only increases'.format(table_name, gsi_name))
update_gsi_provisioning(
table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=True)
def update_throughput(table_name, read_units, write_units, key_name):
""" Update throughput on the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:type read_units: int
:param read_units: New read unit provisioning
:type write_units: int
:param write_units: New write unit provisioning
:type key_name: str
:param key_name: Configuration option key name
"""
table = dynamodb.get_table(table_name)
# Check that we are in the right time frame
if get_table_option(key_name, 'maintenance_windows'):
if not __is_maintenance_window(table_name,
get_table_option(key_name, 'maintenance_windows')):
logger.warning(
'{0} - Current time is outside maintenance window'.format(
table_name))
return
else:
logger.info(
'{0} - Current time is within maintenance window'.format(
table_name))
# Check table status
if table.status != 'ACTIVE':
logger.warning(
'{0} - Not performing throughput changes when table '
'is in {1} state'.format(table_name, table.status))
# If this setting is True, we will only scale down when
# BOTH reads AND writes are low
if get_table_option(key_name, 'always_decrease_rw_together'):
if (read_units < table.read_units) or (table.read_units == get_table_option(key_name, 'min_provisioned_reads')):
if (write_units < table.write_units) or (table.write_units == get_table_option(key_name, 'min_provisioned_writes')):
logger.info(
'{0} - Both reads and writes will be decreased'.format(
table_name))
elif read_units < table.read_units:
logger.info(
'{0} - Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(table_name))
read_units = table.read_units
elif write_units < table.write_units:
logger.info(
'{0} - Will not decrease reads nor writes, waiting for '
'both to become low before decrease'.format(table_name))
write_units = table.write_units
if read_units == table.read_units and write_units == table.write_units:
logger.debug('{0} - No need to update provisioning')
return
if not get_global_option('dry_run'):
try:
table.update_throughput(int(read_units), int(write_units))
logger.info('Provisioning updated')
except DynamoDBResponseError as error:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'LimitExceededException':
logger.warning(
'{0} - {1}'.format(table_name, error.body['message']))
if int(read_units) > table.read_units:
logger.info('{0} - Scaling up reads to {1:d}'.format(
table_name,
int(read_units)))
update_throughput(
table_name,
int(read_units),
int(table.write_units),
key_name)
elif int(write_units) > table.write_units:
logger.info('{0} - Scaling up writes to {1:d}'.format(
table_name,
int(write_units)))
update_throughput(
table_name,
int(table.read_units),
int(write_units),
key_name)
elif dynamodb_error == 'ValidationException':
logger.warning('{0} - ValidationException: {1}'.format(
table_name,
error.body['message']))
elif dynamodb_error == 'ResourceInUseException':
logger.warning('{0} - ResourceInUseException: {1}'.format(
table_name,
error.body['message']))
elif dynamodb_error == 'AccessDeniedException':
logger.warning('{0} - AccessDeniedException: {1}'.format(
table_name,
error.body['message']))
else:
logger.error(
(
'{0} - Unhandled exception: {1}: {2}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name,
dynamodb_error,
error.body['message']))
def update_gsi_provisioning(
table_name, table_key, gsi_name, gsi_key,
reads, writes, retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
if retry_with_only_increase:
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits": reads,
"WriteCapacityUnits": writes
}
}
}
])
message = (
'{0} - GSI: {1} - Provisioning updated to '
'{2} reads and {3} writes').format(
table_name, gsi_name, reads, writes)
logger.info(message)
# See if we should send notifications for scale-down, scale-up or both
sns_message_types = []
if current_reads > reads or current_writes > current_writes:
sns_message_types.append('scale-down')
if current_reads < reads or current_writes < current_writes:
sns_message_types.append('scale-up')
sns.publish_gsi_notification(
table_key,
gsi_key,
message,
sns_message_types,
subject='Updated provisioning for GSI {0}'.format(gsi_name))
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name, gsi_name, exception, error.body['message']))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - GSI: {1} - Will retry to update provisioning '
'with only increases'.format(table_name, gsi_name))
update_gsi_provisioning(
table_name,
table_key,
gsi_name,
gsi_key,
reads,
writes,
retry_with_only_increase=True)
def update_gsi_provisioning(
table_name, gsi_name, reads, writes, retry_with_only_increase=False):
""" Update provisioning on a global secondary index
:type table_name: str
:param table_name: Name of the table
:type gsi_name: str
:param gsi_name: Name of the GSI
:type reads: int
:param reads: Number of reads to provision
:type writes: int
:param writes: Number of writes to provision
:type retry_with_only_increase: bool
:param retry_with_only_increase: Set to True to ensure only increases
"""
if retry_with_only_increase:
current_reads = int(get_provisioned_table_read_units(table_name))
current_writes = int(get_provisioned_table_write_units(table_name))
# Ensure that we are only doing increases
if current_reads > reads:
reads = current_reads
if current_writes > writes:
writes = current_writes
logger.info(
'{0} - Retrying to update provisioning, excluding any decreases. '
'Setting new reads to {1} and new writes to {2}'.format(
table_name, reads, writes))
try:
DYNAMODB_CONNECTION.update_table(
table_name=table_name,
global_secondary_index_updates=[
{
"Update": {
"IndexName": gsi_name,
"ProvisionedThroughput": {
"ReadCapacityUnits": reads,
"WriteCapacityUnits": writes
}
}
}
])
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
know_exceptions = ['LimitExceededException']
if exception in know_exceptions:
logger.warning('{0} - GSI: {1} - {2}: {3}'.format(
table_name, gsi_name, exception, error.body['message']))
else:
logger.error(
(
'{0} - GSI: {1} - Unhandled exception: {2}: {3}. '
'Please file a bug report at '
'https://github.com/sebdah/dynamic-dynamodb/issues'
).format(
table_name, gsi_name, exception, error.body['message']))
if (not retry_with_only_increase and
exception == 'LimitExceededException'):
logger.info(
'{0} - GSI: {1} - Will retry to update provisioning '
'with only increases'.format(table_name, gsi_name))
update_gsi_provisioning(
table_name,
gsi_name,
reads,
writes,
retry_with_only_increase=True)
def ensure_provisioning(table_name, table_key, gsi_name, gsi_key,
num_consec_read_checks, num_consec_write_checks):
""" Ensure that provisioning is correct for Global Secondary Indexes
:type table_name: str
:param table_name: Name of the DynamoDB table
:type table_key: str
:param table_key: Table configuration option key name
:type gsi_name: str
:param gsi_name: Name of the GSI
:type gsi_key: str
:param gsi_key: GSI configuration option key name
:type num_consec_read_checks: int
:param num_consec_read_checks: How many consecutive checks have we had
:type num_consec_write_checks: int
:param num_consec_write_checks: How many consecutive checks have we had
:returns: (int, int) -- num_consec_read_checks, num_consec_write_checks
"""
if get_global_option('circuit_breaker_url'):
if circuit_breaker.is_open():
logger.warning('Circuit breaker is OPEN!')
return (0, 0)
logger.info(
'{0} - Will ensure provisioning for global secondary index {1}'.format(
table_name, gsi_name))
try:
read_update_needed, updated_read_units, num_consec_read_checks = \
__ensure_provisioning_reads(
table_name,
table_key,
gsi_name,
gsi_key,
num_consec_read_checks)
write_update_needed, updated_write_units, num_consec_write_checks = \
__ensure_provisioning_writes(
table_name,
table_key,
gsi_name,
gsi_key,
num_consec_write_checks)
# Handle throughput updates
if read_update_needed or write_update_needed:
logger.info('{0} - GSI: {1} - Changing provisioning to {2:d} '
'read units and {3:d} write units'.format(
table_name, gsi_name, int(updated_read_units),
int(updated_write_units)))
__update_throughput(table_name, table_key, gsi_name, gsi_key,
updated_read_units, updated_write_units)
else:
logger.info(
'{0} - GSI: {1} - No need to change provisioning'.format(
table_name, gsi_name))
except JSONResponseError:
raise
except BotoServerError:
raise
return num_consec_read_checks, num_consec_write_checks
