def _get_secs_since_increase(self, boto_table): default_timestamp = 0.0 description = boto_table.describe() timestamp = (description['Table']['ProvisionedThroughput'].get( 'LastIncreaseDateTime', default_timestamp)) if timestamp == default_timestamp: logging.warn('Unable to determine LastIncreaseDateTime for table') last_modified = datetime.fromtimestamp(timestamp) secs_since_increase = (datetime.now() - last_modified).total_seconds() return secs_since_increase
def handle_failed_migration(db, exc_info): logging.error('Aborting migrations - ' 'Failed with: %s %s %s', exc_info[0], exc_info[1], exc_info[2]) if ALLOW_RETRYING_MIGRATIONS: release_migration_lock(db) logging.warn('Migrations will retry next server start ' '"%s" to "%s" in the "%s" table' % ( MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME)) else: logging.warn('You will need to set ' '"%s" to "%s" in the "%s" table' % ( MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME))
def handle_failed_migration(db, exc_info): logging.error('Aborting migrations - ' 'Failed with: %s %s %s', exc_info[0], exc_info[1], exc_info[2]) if ALLOW_RETRYING_MIGRATIONS: release_migration_lock(db) logging.warn('Migrations will retry next server start ' '"%s" to "%s" in the "%s" table' % (MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME)) else: logging.warn('You will need to set ' '"%s" to "%s" in the "%s" table' % (MIGRATION_META_STATUS_KEY, NO_STATUS, META_TABLE_NAME))
def _change_throughput(self, new_throughput, boto_table, table_name): try: logging.warn('Attempting to increase throughput of ' + table_name) self.connection.update_table(table_name, provisioned_throughput=new_throughput) except Exception as e: # TODO: Fail gracefully here on Validation Exception. # TODO: Don't refresh table info after getting throughput exceeded exc_info = sys.exc_info() logging.error( 'Could not increase table throughput will continue ' 'retrying. Error was: %s %s %s', exc_info[0], exc_info[1], exc_info[2]) else: logging.info('Successfully increased throughput of ' + table_name)
def list_tables(self): table_names = self.connection.list_tables()['TableNames'] table_data = {} for table_name in table_names: try: description = self.connection.describe_table( table_name)['Table'] except Exception as e: if e.error_code.find('ResourceNotFoundException') >= 0: logging.warn('Table ' + table_name + ' was just deleted, cannot describe.') else: raise e else: table_data[table_name] = description return table_data
def _handle_throughput_exceeded(self, new_throughput, retry, boto_table): logging.warn('ProvisionedThroughputExceededException retrying: ' + str(retry)) if retry == 0: # Only increase throughput on first retry for this request. # assert False # TODO: Create our own last_increase_time in meta unless AWS fixes # their ProvisionedThroughputDescription response. # TODO: See if throughput increased. secs_since_increase = self._get_secs_since_increase(boto_table) if secs_since_increase > 5: if self.get_table_status(boto_table) != 'UPDATING': # Avoid piling on throughput from several requests self._change_throughput(new_throughput, boto_table, boto_table.table_name) self._exponential_splay(retry)