def _get_secs_since_increase(self, boto_table):
default_timestamp = 0.0
description = boto_table.describe()
timestamp = (description['Table']['ProvisionedThroughput'].get(
'LastIncreaseDateTime', default_timestamp))
if timestamp == default_timestamp:
logging.warn('Unable to determine LastIncreaseDateTime for table')
last_modified = datetime.fromtimestamp(timestamp)
secs_since_increase = (datetime.now() - last_modified).total_seconds()
return secs_since_increase
def handle_failed_migration(db, exc_info):
logging.error('Aborting migrations - '
'Failed with: %s %s %s',
exc_info[0], exc_info[1], exc_info[2])
if ALLOW_RETRYING_MIGRATIONS:
release_migration_lock(db)
logging.warn('Migrations will retry next server start '
'"%s" to "%s" in the "%s" table' % (
MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME))
else:
logging.warn('You will need to set '
'"%s" to "%s" in the "%s" table' % (
MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME))
def handle_failed_migration(db, exc_info):
logging.error('Aborting migrations - '
'Failed with: %s %s %s', exc_info[0], exc_info[1],
exc_info[2])
if ALLOW_RETRYING_MIGRATIONS:
release_migration_lock(db)
logging.warn('Migrations will retry next server start '
'"%s" to "%s" in the "%s" table' %
(MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME))
else:
logging.warn('You will need to set '
'"%s" to "%s" in the "%s" table' %
(MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME))
def _change_throughput(self, new_throughput, boto_table, table_name):
try:
logging.warn('Attempting to increase throughput of ' + table_name)
self.connection.update_table(table_name,
provisioned_throughput=new_throughput)
except Exception as e:
# TODO: Fail gracefully here on Validation Exception.
# TODO: Don't refresh table info after getting throughput exceeded
exc_info = sys.exc_info()
logging.error(
'Could not increase table throughput will continue '
'retrying. Error was: %s %s %s', exc_info[0], exc_info[1],
exc_info[2])
else:
logging.info('Successfully increased throughput of ' + table_name)
def list_tables(self):
table_names = self.connection.list_tables()['TableNames']
table_data = {}
for table_name in table_names:
try:
description = self.connection.describe_table(
table_name)['Table']
except Exception as e:
if e.error_code.find('ResourceNotFoundException') >= 0:
logging.warn('Table ' + table_name +
' was just deleted, cannot describe.')
else:
raise e
else:
table_data[table_name] = description
return table_data
def _handle_throughput_exceeded(self, new_throughput, retry, boto_table):
logging.warn('ProvisionedThroughputExceededException retrying: ' +
str(retry))
if retry == 0:
# Only increase throughput on first retry for this request.
# assert False
# TODO: Create our own last_increase_time in meta unless AWS fixes
# their ProvisionedThroughputDescription response.
# TODO: See if throughput increased.
secs_since_increase = self._get_secs_since_increase(boto_table)
if secs_since_increase > 5:
if self.get_table_status(boto_table) != 'UPDATING':
# Avoid piling on throughput from several requests
self._change_throughput(new_throughput, boto_table,
boto_table.table_name)
self._exponential_splay(retry)